Coverage report

We generally consider things that are slower on PyPy than CPython to be bugs of PyPy. If you find some issue that is not documented here, -please report it to our bug tracker for investigation.

diff --git a/source/performance.txt b/source/performance.txt --- a/source/performance.txt +++ b/source/performance.txt @@ -327,6 +327,6 @@ of PyPy. If you find some issue that is not documented here, please report it to our `bug tracker`_ for investigation. -.. _`bug tracker`: http://bugs.pypy.org +.. _`bug tracker`: https://bitbucket.org/pypy/pypy/issues?status=new&status=open .. _`speed website`: http://speed.pypy.org .. _CFFI: http://cffi.readthedocs.org/ From noreply at buildbot.pypy.org Thu Apr 2 15:16:19 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 2 Apr 2015 15:16:19 +0200 (CEST) Subject: [pypy-commit] pypy vmprof: Broken logic: every 1000 code objects, one was ignored Message-ID: <20150402131619.F38091C117D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof Changeset: r76684:2b41c53c69cb Date: 2015-04-02 15:09 +0200 http://bitbucket.org/pypy/pypy/changeset/2b41c53c69cb/ Log: Broken logic: every 1000 code objects, one was ignored diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py --- a/pypy/module/_vmprof/interp_vmprof.py +++ b/pypy/module/_vmprof/interp_vmprof.py @@ -181,9 +181,8 @@ if self.fileno == -1: raise OperationError(space.w_RuntimeError, space.wrap("vmprof not running")) - if len(self.current_codes) < MAX_CODES: - self.current_codes.append(code) - else: + self.current_codes.append(code) + if len(self.current_codes) >= MAX_CODES: self._flush_codes(space) def _flush_codes(self, space): From noreply at buildbot.pypy.org Thu Apr 2 15:16:21 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 2 Apr 2015 15:16:21 +0200 (CEST) Subject: [pypy-commit] pypy vmprof: Reduce the diff Message-ID: <20150402131621.32DC51C117D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof Changeset: r76685:46fbc0de3139 Date: 2015-04-02 15:12 +0200 http://bitbucket.org/pypy/pypy/changeset/46fbc0de3139/ Log: Reduce the diff diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1,8 +1,8 @@ -import sys, os +import sys from rpython.rlib.cache import Cache from rpython.tool.uid import HUGEVAL_BYTES -from rpython.rlib import jit, types, rfile +from rpython.rlib import jit, types from rpython.rlib.debug import make_sure_not_resized from rpython.rlib.objectmodel import (we_are_translated, newlist_hint, compute_unique_id, specialize) diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -130,4 +130,3 @@ def locals_to_fast(space, w_frame): assert isinstance(w_frame, PyFrame) w_frame.locals2fast() - From noreply at buildbot.pypy.org Thu Apr 2 15:18:28 2015 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 2 Apr 2015 15:18:28 +0200 (CEST) Subject: [pypy-commit] pypy vmprof: fix the test Message-ID: <20150402131828.2D3B91C076F@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76686:2418e4a66b75 Date: 2015-04-02 15:17 +0200 http://bitbucket.org/pypy/pypy/changeset/2418e4a66b75/ Log: fix the test diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py --- a/pypy/module/_vmprof/test/test__vmprof.py +++ b/pypy/module/_vmprof/test/test__vmprof.py @@ -21,6 +21,11 @@ i = 0 count = 0 i += 5 * WORD # header + assert s[i] == '\x04' + i += 1 # marker + assert s[i] == '\x04' + i += 1 # length + i += len('pypy') while i < len(s): assert s[i] == '\x02' i += 1 From noreply at buildbot.pypy.org Thu Apr 2 15:18:29 2015 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 2 Apr 2015 15:18:29 +0200 (CEST) Subject: [pypy-commit] pypy vmprof: merge Message-ID: <20150402131829.5A6241C076F@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76687:dc181afb0e0e Date: 2015-04-02 15:18 +0200 http://bitbucket.org/pypy/pypy/changeset/dc181afb0e0e/ Log: merge diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1,8 +1,8 @@ -import sys, os +import sys from rpython.rlib.cache import Cache from rpython.tool.uid import HUGEVAL_BYTES -from rpython.rlib import jit, types, rfile +from rpython.rlib import jit, types from rpython.rlib.debug import make_sure_not_resized from rpython.rlib.objectmodel import (we_are_translated, newlist_hint, compute_unique_id, specialize) diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -130,4 +130,3 @@ def locals_to_fast(space, w_frame): assert isinstance(w_frame, PyFrame) w_frame.locals2fast() - diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py --- a/pypy/module/_vmprof/interp_vmprof.py +++ b/pypy/module/_vmprof/interp_vmprof.py @@ -181,9 +181,8 @@ if self.fileno == -1: raise OperationError(space.w_RuntimeError, space.wrap("vmprof not running")) - if len(self.current_codes) < MAX_CODES: - self.current_codes.append(code) - else: + self.current_codes.append(code) + if len(self.current_codes) >= MAX_CODES: self._flush_codes(space) def _flush_codes(self, space): From noreply at buildbot.pypy.org Thu Apr 2 16:15:51 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 2 Apr 2015 16:15:51 +0200 (CEST) Subject: [pypy-commit] pypy vmprof: Revert 2f9e91643e10: it's wrongish and it's not needed any more Message-ID: <20150402141551.C6E461C1016@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof Changeset: r76688:6b7f4d1bccce Date: 2015-04-02 16:15 +0200 http://bitbucket.org/pypy/pypy/changeset/6b7f4d1bccce/ Log: Revert 2f9e91643e10: it's wrongish and it's not needed any more diff --git a/rpython/translator/c/node.py b/rpython/translator/c/node.py --- a/rpython/translator/c/node.py +++ b/rpython/translator/c/node.py @@ -126,9 +126,8 @@ return self.prefix + name def verbatim_field_name(self, name): - if name.startswith('c_'): # produced in this way by rffi - return name[2:] - return name + assert name.startswith('c_') # produced in this way by rffi + return name[2:] def c_struct_field_type(self, name): return self.STRUCT._flds[name] From noreply at buildbot.pypy.org Thu Apr 2 16:48:05 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Thu, 2 Apr 2015 16:48:05 +0200 (CEST) Subject: [pypy-commit] pypy refactor-pycall: cleanup Message-ID: <20150402144805.44A2B1C1331@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: refactor-pycall Changeset: r76690:ca74b99d487b Date: 2015-04-02 15:47 +0100 http://bitbucket.org/pypy/pypy/changeset/ca74b99d487b/ Log: cleanup diff --git a/rpython/rtyper/normalizecalls.py b/rpython/rtyper/normalizecalls.py --- a/rpython/rtyper/normalizecalls.py +++ b/rpython/rtyper/normalizecalls.py @@ -86,12 +86,7 @@ return False # nothing to do, all signatures already match shape_cnt, shape_keys, shape_star = shape - if shape_star: - raise TyperError( - "not implemented: a call is done with a '*' argument, and the" - " multiple functions or methods that it can go to don't have" - " all the same signature (different argument names or defaults)." - " The call can go to:\n%s" % '\n'.join(map(repr, graphs))) + assert not shape_star, "should have been removed at this stage" # for the first 'shape_cnt' arguments we need to generalize to # a common type diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py --- a/rpython/rtyper/rbuiltin.py +++ b/rpython/rtyper/rbuiltin.py @@ -47,31 +47,14 @@ # to it. return (self.__class__, self.methodname, id(self.s_self)) -def call_args_expand(hop, takes_kwds = True): +def call_args_expand(hop): hop = hop.copy() from rpython.annotator.argument import ArgumentsForTranslation arguments = ArgumentsForTranslation.fromshape( hop.args_s[1].const, # shape range(hop.nb_args-2)) - if arguments.w_stararg is not None: - # expand the *arg in-place -- it must be a tuple - from rpython.rtyper.rtuple import TupleRepr - if arguments.w_stararg != hop.nb_args - 3: - raise TyperError("call pattern too complex") - v_tuple = hop.args_v.pop() - s_tuple = hop.args_s.pop() - r_tuple = hop.args_r.pop() - if not isinstance(r_tuple, TupleRepr): - raise TyperError("*arg must be a tuple") - for i in range(len(r_tuple.items_r)): - v_item = r_tuple.getitem_internal(hop.llops, v_tuple, i) - hop.args_v.append(v_item) - hop.args_s.append(s_tuple.items[i]) - hop.args_r.append(r_tuple.items_r[i]) - + assert arguments.w_stararg is None keywords = arguments.keywords - if not takes_kwds and keywords: - raise TyperError("kwds args not supported") # prefix keyword arguments with 'i_' kwds_i = {} for key in keywords: diff --git a/rpython/rtyper/rptr.py b/rpython/rtyper/rptr.py --- a/rpython/rtyper/rptr.py +++ b/rpython/rtyper/rptr.py @@ -1,4 +1,3 @@ -from rpython.annotator import model as annmodel from rpython.rtyper.llannotation import ( SomePtr, SomeInteriorPtr, SomeLLADTMeth, lltype_to_annotation) from rpython.flowspace import model as flowmodel @@ -107,11 +106,7 @@ resulttype = self.lowleveltype.TO.RESULT) def rtype_call_args(self, hop): - from rpython.rtyper.rbuiltin import call_args_expand - hop, _ = call_args_expand(hop, takes_kwds=False) - hop.swap_fst_snd_args() - hop.r_s_popfirstarg() - return self.rtype_simple_call(hop) + raise TyperError("kwds args not supported") class __extend__(pairtype(PtrRepr, PtrRepr)): def convert_from_to((r_ptr1, r_ptr2), v, llop): diff --git a/rpython/rtyper/test/test_normalizecalls.py b/rpython/rtyper/test/test_normalizecalls.py --- a/rpython/rtyper/test/test_normalizecalls.py +++ b/rpython/rtyper/test/test_normalizecalls.py @@ -237,25 +237,6 @@ import re assert re.match(msg, excinfo.value.args[0]) - def test_methods_with_named_arg_call(self): - class Base: - def fn(self, y): - raise NotImplementedError - class Sub1(Base): - def fn(self, y): - return 1 + y - class Sub2(Base): - def fn(self, x): # different name! - return x - 2 - def dummyfn(n): - if n == 1: - s = Sub1() - else: - s = Sub2() - return s.fn(*(n,)) - - py.test.raises(TyperError, self.rtype, dummyfn, [int], int) - class PBase: def fn(self): From noreply at buildbot.pypy.org Thu Apr 2 16:48:04 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Thu, 2 Apr 2015 16:48:04 +0200 (CEST) Subject: [pypy-commit] pypy refactor-pycall: Implement *-unpacking as an online transform Message-ID: <20150402144804.263B01C1331@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: refactor-pycall Changeset: r76689:0e6061661c1e Date: 2015-04-02 05:06 +0100 http://bitbucket.org/pypy/pypy/changeset/0e6061661c1e/ Log: Implement *-unpacking as an online transform f(*(a, b, c)) should now be completely equivalent to f(a, b, c) diff --git a/rpython/annotator/argument.py b/rpython/annotator/argument.py --- a/rpython/annotator/argument.py +++ b/rpython/annotator/argument.py @@ -155,18 +155,6 @@ keywords_w = [_kwds_w[key] for key in self.keywords] return ArgumentsForTranslation(args_w, dict(zip(self.keywords, keywords_w))) - @classmethod - def fromshape(cls, (shape_cnt, shape_keys, shape_star), data_w): - args_w = data_w[:shape_cnt] - p = end_keys = shape_cnt + len(shape_keys) - if shape_star: - w_star = data_w[p] - p += 1 - else: - w_star = None - return cls(args_w, dict(zip(shape_keys, data_w[shape_cnt:end_keys])), - w_star) - def rawshape(args): return args._rawshape() diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py --- a/rpython/annotator/test/test_annrpython.py +++ b/rpython/annotator/test/test_annrpython.py @@ -354,6 +354,25 @@ assert isinstance(s, annmodel.SomeInteger) assert s.const == 3 + def test_star_unpack_list(self): + def g(): + pass + def f(l): + return g(*l) + a = self.RPythonAnnotator() + with py.test.raises(annmodel.AnnotatorError): + a.build_types(f, [[int]]) + + def test_star_unpack_and_keywords(self): + def g(a, b, c=0, d=0): + return a + b + c + d + + def f(a, b): + return g(a, *(b,), d=5) + a = self.RPythonAnnotator() + s_result = a.build_types(f, [int, int]) + assert isinstance(s_result, annmodel.SomeInteger) + def test_pbc_attr_preserved_on_instance(self): a = self.RPythonAnnotator() s = a.build_types(snippet.preserve_pbc_attr_on_instance, [bool]) diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py --- a/rpython/annotator/unaryop.py +++ b/rpython/annotator/unaryop.py @@ -6,6 +6,7 @@ from rpython.flowspace.operation import op from rpython.flowspace.model import const, Constant +from rpython.flowspace.argument import CallSpec from rpython.annotator.model import (SomeObject, SomeInteger, SomeBool, SomeString, SomeChar, SomeList, SomeDict, SomeTuple, SomeImpossibleValue, SomeUnicodeCodePoint, SomeInstance, SomeBuiltin, SomeBuiltinMethod, @@ -47,11 +48,35 @@ @op.simple_call.register(SomeObject) def simple_call_SomeObject(annotator, func, *args): - return annotator.annotation(func).call(simple_args([annotator.annotation(arg) for arg in args])) + return annotator.annotation(func).call( + simple_args([annotator.annotation(arg) for arg in args])) + + at op.call_args.register_transform(SomeObject) +def transform_varargs(annotator, v_func, v_shape, *data_v): + callspec = CallSpec.fromshape(v_shape.value, list(data_v)) + v_vararg = callspec.w_stararg + if callspec.w_stararg: + s_vararg = annotator.annotation(callspec.w_stararg) + if not isinstance(s_vararg, SomeTuple): + raise AnnotatorError( + "Calls like f(..., *arg) require 'arg' to be a tuple") + n_items = len(s_vararg.items) + ops = [op.getitem(v_vararg, const(i)) for i in range(n_items)] + new_args = callspec.arguments_w + [hlop.result for hlop in ops] + if callspec.keywords: + newspec = CallSpec(new_args, callspec.keywords) + shape, data_v = newspec.flatten() + call_op = op.call_args(v_func, const(shape), *data_v) + else: + call_op = op.simple_call(v_func, *new_args) + ops.append(call_op) + return ops + @op.call_args.register(SomeObject) -def call_args(annotator, func, *args): - return annotator.annotation(func).call(complex_args([annotator.annotation(arg) for arg in args])) +def call_args(annotator, func, *args_v): + callspec = complex_args([annotator.annotation(v_arg) for v_arg in args_v]) + return annotator.annotation(func).call(callspec) class __extend__(SomeObject): @@ -722,7 +747,7 @@ if attr not in dct: continue obj = dct[attr] - if (not isinstance(obj, Constant) or + if (not isinstance(obj, Constant) or not isinstance(obj.value, property)): return result.append(getattr(obj.value, meth)) diff --git a/rpython/flowspace/argument.py b/rpython/flowspace/argument.py --- a/rpython/flowspace/argument.py +++ b/rpython/flowspace/argument.py @@ -111,3 +111,16 @@ return self.arguments_w else: return self.arguments_w + [const(x) for x in self.w_stararg.value] + + @classmethod + def fromshape(cls, (shape_cnt, shape_keys, shape_star), data_w): + args_w = data_w[:shape_cnt] + p = end_keys = shape_cnt + len(shape_keys) + if shape_star: + w_star = data_w[p] + p += 1 + else: + w_star = None + return cls(args_w, dict(zip(shape_keys, data_w[shape_cnt:end_keys])), + w_star) + From noreply at buildbot.pypy.org Thu Apr 2 22:26:46 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Thu, 2 Apr 2015 22:26:46 +0200 (CEST) Subject: [pypy-commit] pypy default: Issue2014: Add PyLong_FromUnicode to cpyext. Message-ID: <20150402202646.7FBA01C076F@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r76691:e5d47d0b27b8 Date: 2015-04-02 22:26 +0200 http://bitbucket.org/pypy/pypy/changeset/e5d47d0b27b8/ Log: Issue2014: Add PyLong_FromUnicode to cpyext. diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py --- a/pypy/module/cpyext/longobject.py +++ b/pypy/module/cpyext/longobject.py @@ -186,6 +186,16 @@ pend[0] = rffi.ptradd(str, len(s)) return space.call_function(space.w_long, w_str, w_base) + at cpython_api([rffi.CWCHARP, Py_ssize_t, rffi.INT_real], PyObject) +def PyLong_FromUnicode(space, u, length, base): + """Convert a sequence of Unicode digits to a Python long integer value. + The first parameter, u, points to the first character of the Unicode + string, length gives the number of characters, and base is the radix + for the conversion. The radix must be in the range [2, 36]; if it is + out of range, ValueError will be raised.""" + w_value = space.wrap(rffi.wcharpsize2unicode(u, length)) + return space.call_function(space.w_long, w_value) + @cpython_api([rffi.VOIDP], PyObject) def PyLong_FromVoidPtr(space, p): """Create a Python integer or long integer from the pointer p. The pointer value diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py --- a/pypy/module/cpyext/stubs.py +++ b/pypy/module/cpyext/stubs.py @@ -1395,18 +1395,6 @@ """ raise NotImplementedError - at cpython_api([rffi.CWCHARP, Py_ssize_t, rffi.INT_real], PyObject) -def PyLong_FromUnicode(space, u, length, base): - """Convert a sequence of Unicode digits to a Python long integer value. The first - parameter, u, points to the first character of the Unicode string, length - gives the number of characters, and base is the radix for the conversion. The - radix must be in the range [2, 36]; if it is out of range, ValueError - will be raised. - - This function used an int for length. This might require - changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([PyObject, rffi.CCHARP], rffi.INT_real, error=-1) def PyMapping_DelItemString(space, o, key): """Remove the mapping for object key from the object o. Return -1 on diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py --- a/pypy/module/cpyext/test/test_longobject.py +++ b/pypy/module/cpyext/test/test_longobject.py @@ -180,3 +180,14 @@ assert module.from_bytearray(False, False) == 0xBC9A assert module.from_bytearray(False, True) == -0x4365 + def test_fromunicode(self): + module = self.import_extension('foo', [ + ("from_unicode", "METH_O", + """ + Py_UNICODE* u = PyUnicode_AsUnicode(args); + return PyLong_FromUnicode(u, 6, 10); + """), + ]) + # A string with arabic digits. 'BAD' is after the 6th character. + assert module.from_unicode(u' 1\u0662\u0663\u0664BAD') == 1234 + From noreply at buildbot.pypy.org Thu Apr 2 22:35:01 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Thu, 2 Apr 2015 22:35:01 +0200 (CEST) Subject: [pypy-commit] pypy default: PyLong_FromUnicode: Add support for the "base" parameter. Message-ID: <20150402203501.443811C076F@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r76692:d8d5f2097351 Date: 2015-04-02 22:35 +0200 http://bitbucket.org/pypy/pypy/changeset/d8d5f2097351/ Log: PyLong_FromUnicode: Add support for the "base" parameter. diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py --- a/pypy/module/cpyext/longobject.py +++ b/pypy/module/cpyext/longobject.py @@ -194,7 +194,8 @@ for the conversion. The radix must be in the range [2, 36]; if it is out of range, ValueError will be raised.""" w_value = space.wrap(rffi.wcharpsize2unicode(u, length)) - return space.call_function(space.w_long, w_value) + w_base = space.wrap(rffi.cast(lltype.Signed, base)) + return space.call_function(space.w_long, w_value, w_base) @cpython_api([rffi.VOIDP], PyObject) def PyLong_FromVoidPtr(space, p): diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py --- a/pypy/module/cpyext/test/test_longobject.py +++ b/pypy/module/cpyext/test/test_longobject.py @@ -185,9 +185,11 @@ ("from_unicode", "METH_O", """ Py_UNICODE* u = PyUnicode_AsUnicode(args); - return PyLong_FromUnicode(u, 6, 10); + return Py_BuildValue("NN", + PyLong_FromUnicode(u, 6, 10), + PyLong_FromUnicode(u, 6, 16)); """), ]) # A string with arabic digits. 'BAD' is after the 6th character. - assert module.from_unicode(u' 1\u0662\u0663\u0664BAD') == 1234 + assert module.from_unicode(u' 1\u0662\u0663\u0664BAD') == (1234, 4660) From noreply at buildbot.pypy.org Fri Apr 3 03:23:37 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Fri, 3 Apr 2015 03:23:37 +0200 (CEST) Subject: [pypy-commit] pypy refactor-pycall: cleanup Message-ID: <20150403012337.470651C1016@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: refactor-pycall Changeset: r76693:4f23a232ec40 Date: 2015-04-03 02:22 +0100 http://bitbucket.org/pypy/pypy/changeset/4f23a232ec40/ Log: cleanup diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py --- a/rpython/annotator/annrpython.py +++ b/rpython/annotator/annrpython.py @@ -10,7 +10,6 @@ Variable, Constant, FunctionGraph, checkgraph) from rpython.translator import simplify, transform from rpython.annotator import model as annmodel, signature -from rpython.annotator.argument import simple_args from rpython.annotator.bookkeeper import Bookkeeper from rpython.rtyper.normalizecalls import perform_normalizations @@ -91,14 +90,11 @@ def get_call_parameters(self, function, args_s, policy): desc = self.bookkeeper.getdesc(function) - from rpython.annotator.description import FunctionDesc - assert isinstance(desc, FunctionDesc) - args = simple_args(args_s) prevpolicy = self.policy self.policy = policy self.bookkeeper.enter(None) try: - return desc.pycall_2(args) + return desc.get_call_parameters(args_s) finally: self.bookkeeper.leave() self.policy = prevpolicy diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py --- a/rpython/annotator/description.py +++ b/rpython/annotator/description.py @@ -1,10 +1,10 @@ from __future__ import absolute_import import types from rpython.annotator.signature import ( - enforce_signature_args, enforce_signature_return) + enforce_signature_args, enforce_signature_return, finish_type) from rpython.flowspace.model import Constant, FunctionGraph from rpython.flowspace.bytecode import cpython_code_signature -from rpython.annotator.argument import rawshape, ArgErr +from rpython.annotator.argument import rawshape, ArgErr, simple_args from rpython.tool.sourcetools import valid_identifier, func_with_new_name from rpython.tool.pairtype import extendabletype from rpython.annotator.model import ( @@ -320,7 +320,8 @@ result = unionof(result, s_previous_result) return result - def pycall_2(self, args): + def get_call_parameters(self, args_s): + args = simple_args(args_s) inputcells = self.parse_arguments(args) graph = self.specialize(inputcells) assert isinstance(graph, FunctionGraph) @@ -329,16 +330,13 @@ # recreate the args object because inputcells may have been changed new_args = args.unmatch_signature(self.signature, inputcells) inputcells = self.parse_arguments(new_args, graph) - res = graph, inputcells - result = s_ImpossibleValue signature = getattr(self.pyobj, '_signature_', None) if signature: - sigresult = enforce_signature_return(self, signature[1], result) - if sigresult is not None: + s_result = finish_type(signature[1], self.bookkeeper, self.pyobj) + if s_result is not None: self.bookkeeper.annotator.addpendingblock( - graph, graph.returnblock, [sigresult]) - result = sigresult - return res + graph, graph.returnblock, [s_result]) + return graph, inputcells def bind_under(self, classdef, name): # XXX static methods From noreply at buildbot.pypy.org Fri Apr 3 08:02:47 2015 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 3 Apr 2015 08:02:47 +0200 (CEST) Subject: [pypy-commit] pypy object-dtype2: add many ufuncs Message-ID: <20150403060247.8AE961C1331@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: object-dtype2 Changeset: r76695:f8e674fa7afc Date: 2015-04-03 00:23 +0300 http://bitbucket.org/pypy/pypy/changeset/f8e674fa7afc/ Log: add many ufuncs diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py --- a/pypy/module/micronumpy/test/test_ufuncs.py +++ b/pypy/module/micronumpy/test/test_ufuncs.py @@ -397,7 +397,7 @@ for i in range(3): assert min_c_b[i] == min(b[i], c) - def test_scalar(self): + def test_all_available(self): # tests that by calling all available ufuncs on scalars, none will # raise uncaught interp-level exceptions, (and crash the test) # and those that are uncallable can be accounted for. @@ -412,6 +412,8 @@ if isinstance(u, np.ufunc): try: u(* [array] * u.nin) + except AttributeError: + pass except TypeError: assert s not in uncallable uncallable.add(s) @@ -427,7 +429,9 @@ 'fabs', 'fmod', 'invert', 'mod', 'logaddexp', 'logaddexp2', 'left_shift', 'right_shift', 'copysign', 'signbit', 'ceil', 'floor', 'trunc']) - assert find_uncallable_ufuncs('object') == set() + assert find_uncallable_ufuncs('object') == set( + ['isnan', 'logaddexp2', 'copysign', 'isfinite', 'signbit', + 'isinf', 'logaddexp']) def test_int_only(self): from numpy import bitwise_and, array diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -149,7 +149,9 @@ return self.box(array[0]) def unbox(self, box): - assert isinstance(box, self.BoxType) + if not isinstance(box, self.BoxType): + # i.e. box is an ObjectBox + raise oefmt(self.space.w_AttributeError, '') return box.value def coerce(self, space, dtype, w_item): @@ -427,29 +429,25 @@ def default_fromstring(self, space): return self.box(0) - @specialize.argtype(1, 2) - def div(self, b1, b2): - v1 = self.for_computation(self.unbox(b1)) - v2 = self.for_computation(self.unbox(b2)) + @simple_binary_op + def div(self, v1, v2): if v2 == 0: - return self.box(0) + return 0 if (self.T is rffi.SIGNEDCHAR or self.T is rffi.SHORT or self.T is rffi.INT or self.T is rffi.LONG or self.T is rffi.LONGLONG): if v2 == -1 and v1 == self.for_computation(most_neg_value_of(self.T)): - return self.box(0) - return self.box(v1 / v2) + return 0 + return v1 / v2 - @specialize.argtype(1, 2) - def floordiv(self, b1, b2): - v1 = self.for_computation(self.unbox(b1)) - v2 = self.for_computation(self.unbox(b2)) + @simple_binary_op + def floordiv(self, v1, v2): if v2 == 0: - return self.box(0) + return 0 if (self.T is rffi.SIGNEDCHAR or self.T is rffi.SHORT or self.T is rffi.INT or self.T is rffi.LONG or self.T is rffi.LONGLONG): if v2 == -1 and v1 == self.for_computation(most_neg_value_of(self.T)): - return self.box(0) - return self.box(v1 // v2) + return 0 + return v1 // v2 @simple_binary_op def mod(self, v1, v2): @@ -485,7 +483,7 @@ elif v < 0: return -1 else: - assert v == 0 + #assert v == 0 return 0 @raw_unary_op @@ -1711,16 +1709,134 @@ def for_computation(v): return v - @simple_binary_op - def add(self, v1, v2): - return v1 - #return self.space.add(v1, v2) + @raw_binary_op + def eq(self, v1, v2): + return self.space.eq_w(v1, v2) @raw_binary_op - def eq(self, v1, v2): - return True - #return self.space.eq_w(v1, v2) + def max(self, v1, v2): + if self.space.is_true(self.space.ge(v1, v2)): + return v1 + return v2 + @raw_binary_op + def min(self, v1, v2): + if self.space.is_true(self.space.le(v1, v2)): + return v1 + return v2 + + def arctan2(self, v1, v2): + raise oefmt(self.space.w_AttributeError, 'arctan2') + + @specialize.argtype(1) + def _bool(self, v): + return self.space.bool_w(v) + + @raw_binary_op + def logical_and(self, v1, v2): + return self._bool(v1) and self._bool(v2) + + @raw_binary_op + def logical_or(self, v1, v2): + return self._bool(v1) or self._bool(v2) + + @raw_unary_op + def logical_not(self, v): + return not self._bool(v) + + @raw_binary_op + def logical_xor(self, v1, v2): + a = self._bool(v1) + b = self._bool(v2) + return (not b and a) or (not a and b) + + @simple_binary_op + def bitwise_and(self, v1, v2): + return self.space.and_(v1, v2) + + @simple_binary_op + def bitwise_or(self, v1, v2): + return self.space.or_(v1, v2) + + @simple_binary_op + def bitwise_xor(self, v1, v2): + return self.space.xor(v1, v2) + + @simple_binary_op + def pow(self, v1, v2): + return self.space.pow(v1, v2, self.space.wrap(1)) + + @simple_unary_op + def reciprocal(self, v1): + return self.space.div(self.space.wrap(1.0), v1) + + @simple_unary_op + def sign(self, v): + zero = self.space.wrap(0) + one = self.space.wrap(1) + m_one = self.space.wrap(-1) + if self.space.is_true(self.space.gt(v, zero)): + return one + elif self.space.is_true(self.space.lt(v, zero)): + return m_one + else: + return zero + + +def add_unsupported_op(cls, op): + def func(self, *args): + raise oefmt(self.space.w_TypeError, + "ufunc '%s' not supported for input types", op) + func.__name__ = 'object_' + op + setattr(cls, op, func) + + +def add_unary_op(cls, op): + @simple_unary_op + def func(self, w_v): + w_impl = self.space.lookup(w_v, op) + if w_impl is None: + raise oefmt(self.space.w_AttributeError, op) + return self.space.get_and_call_function(w_impl, w_v) + func.__name__ = 'object_' + op + setattr(cls, op, func) + +def add_space_unary_op(cls, op): + @simple_unary_op + def func(self, v): + return getattr(self.space, op)(v) + func.__name__ = 'object_' + op + setattr(cls, op, func) + +def add_space_binary_op(cls, op): + @simple_binary_op + def func(self, v1, v2): + return getattr(self.space, op)(v1, v2) + func.__name__ = 'object_' + op + setattr(cls, op, func) + +def add_space_comp_op(cls, op): + @raw_binary_op + def func(self, v1, v2): + return self.space.is_true(getattr(self.space, op)(v1, v2)) + func.__name__ = 'object_' + op + setattr(cls, op, func) + +for op in ('copysign', 'isfinite', 'isinf', 'isnan', 'logaddexp', 'logaddexp2', + 'signbit',): + add_unsupported_op(ObjectType, op) +for op in ('conj', 'real', 'imag', 'rint'): + add_unary_op(ObjectType, op) +for op in ('abs', 'neg', 'pos', 'invert'): + add_space_unary_op(ObjectType, op) +for op in ('add', 'floordiv', 'div', 'mod', 'mul', 'sub', 'lshift', 'rshift'): + add_space_binary_op(ObjectType, op) +for op in ('lt', 'gt', 'le', 'ge', 'ne'): + add_space_comp_op(ObjectType, op) + +ObjectType.fmax = ObjectType.max +ObjectType.fmin = ObjectType.min +ObjectType.fmod = ObjectType.mod class FlexibleType(BaseType): def get_element_size(self): diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py --- a/pypy/module/micronumpy/ufuncs.py +++ b/pypy/module/micronumpy/ufuncs.py @@ -349,7 +349,7 @@ if dtype.is_flexible(): raise OperationError(space.w_TypeError, space.wrap('Not implemented for this type')) - if (self.int_only and not dtype.is_int() or + if (self.int_only and not (dtype.is_int() or dtype.is_object()) or not self.allow_bool and dtype.is_bool() or not self.allow_complex and dtype.is_complex()): raise oefmt(space.w_TypeError, @@ -465,9 +465,9 @@ w_ldtype, w_rdtype, promote_to_float=self.promote_to_float, promote_bools=self.promote_bools) - if (self.int_only and (not w_ldtype.is_int() or - not w_rdtype.is_int() or - not calc_dtype.is_int()) or + if (self.int_only and (not (w_ldtype.is_int() or w_ldtype.is_object()) or + not (w_rdtype.is_int() or w_rdtype.is_object()) or + not (calc_dtype.is_int() or calc_dtype.is_object())) or not self.allow_bool and (w_ldtype.is_bool() or w_rdtype.is_bool()) or not self.allow_complex and (w_ldtype.is_complex() or From noreply at buildbot.pypy.org Fri Apr 3 08:02:46 2015 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 3 Apr 2015 08:02:46 +0200 (CEST) Subject: [pypy-commit] pypy object-dtype2: small tweaks to dtype name, customtrace; add (failing) tests Message-ID: <20150403060246.470D91C1331@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: object-dtype2 Changeset: r76694:be19ae2bdd93 Date: 2015-04-01 23:22 +0300 http://bitbucket.org/pypy/pypy/changeset/be19ae2bdd93/ Log: small tweaks to dtype name, customtrace; add (failing) tests diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py --- a/pypy/module/micronumpy/boxes.py +++ b/pypy/module/micronumpy/boxes.py @@ -868,7 +868,7 @@ __len__ = interp2app(W_UnicodeBox.descr_len), ) -W_ObjectBox.typedef = TypeDef("numpy.object", W_ObjectBox.typedef, +W_ObjectBox.typedef = TypeDef("numpy.object_", W_ObjectBox.typedef, __new__ = interp2app(W_ObjectBox.descr__new__.im_func), __getattr__ = interp2app(W_ObjectBox.descr__getattr__), ) diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py --- a/pypy/module/micronumpy/concrete.py +++ b/pypy/module/micronumpy/concrete.py @@ -346,15 +346,12 @@ def customtrace(gc, obj, callback, arg): #debug_print('in customtrace w/obj', obj) - length = rffi.cast(rffi.SIGNEDP, obj + offset_of_length)[0] - step = rffi.cast(rffi.SIGNEDP, obj + offset_of_step)[0] + length = (obj + offset_of_length).signed[0] + step = (obj + offset_of_step).signed[0] storage = (obj + offset_of_storage).address[0] - debug_print('tracing', length, 'objects in ndarray.storage') + #debug_print('tracing', length, 'objects in ndarray.storage') i = 0 while i < length: - #gcref = rffi.cast(llmemory.GCREF, storage) - #w_obj = cast_gcref_to_instance(W_Root, gcref) - #debug_print('tracing', w_obj) gc._trace_callback(callback, arg, storage) storage += step i += 1 @@ -369,7 +366,7 @@ gcstruct = lltype.malloc(OBJECTSTORE) # JIT does not support cast_ptr_to_adr gcstruct.storage = llmemory.cast_ptr_to_adr(storage) - print 'create gcstruct',gcstruct,'with storage',storage,'as',gcstruct.storage + #print 'create gcstruct',gcstruct,'with storage',storage,'as',gcstruct.storage gcstruct.length = length gcstruct.step = elsize return gcstruct diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -1350,42 +1350,4 @@ assert a[0] == 1 assert (a + a)[1] == 4 -class AppTestObjectDtypes(BaseNumpyAppTest): - def test_scalar_from_object(self): - from numpy import array - import sys - class Polynomial(object): - def whatami(self): - return 'an object' - a = array(Polynomial()) - assert a.shape == () - assert a.sum().whatami() == 'an object' - def test_uninitialized_object_array_is_filled_by_None(self): - import numpy as np - - a = np.ndarray([5], dtype="O") - - assert a[0] == None - - def test_object_arrays_add(self): - import numpy as np - - a = np.array(["foo"], dtype=object) - b = np.array(["bar"], dtype=object) - - res = a + b - assert res[0] == "foobar" - - def test_keep_object_alive(self): - # XXX how can I run this test? - import numpy as np - import gc - class O(object): - def whatami(self): - return 'an object' - fiveOs = [O()] * 5 - a = np.array(fiveOs, dtype=object) - del fiveOs - gc.collect() - assert a[2].whatami() == 'an object' diff --git a/pypy/module/micronumpy/test/test_object_arrays.py b/pypy/module/micronumpy/test/test_object_arrays.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/test/test_object_arrays.py @@ -0,0 +1,53 @@ +from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest + + +class AppTestObjectDtypes(BaseNumpyAppTest): + def test_scalar_from_object(self): + from numpy import array + import sys + class Polynomial(object): + def whatami(self): + return 'an object' + a = array(Polynomial()) + assert a.shape == () + assert a.sum().whatami() == 'an object' + + def test_uninitialized_object_array_is_filled_by_None(self): + import numpy as np + + a = np.ndarray([5], dtype="O") + + assert a[0] == None + + def test_object_arrays_add(self): + import numpy as np + + a = np.array(["foo"], dtype=object) + b = np.array(["bar"], dtype=object) + + res = a + b + assert res[0] == "foobar" + + def test_reduce(self): + import numpy as np + class O(object): + def whatami(self): + return 'an object' + fiveOs = [O()] * 5 + a = np.array(fiveOs, dtype=object) + print np.maximum + b = np.maximum.reduce(a) + assert b is not None + + def test_keep_object_alive(self): + # only translated does it really test the gc + import numpy as np + import gc + class O(object): + def whatami(self): + return 'an object' + fiveOs = [O()] * 5 + a = np.array(fiveOs, dtype=object) + del fiveOs + gc.collect() + assert a[2].whatami() == 'an object' diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py --- a/pypy/module/micronumpy/test/test_ufuncs.py +++ b/pypy/module/micronumpy/test/test_ufuncs.py @@ -401,7 +401,7 @@ # tests that by calling all available ufuncs on scalars, none will # raise uncaught interp-level exceptions, (and crash the test) # and those that are uncallable can be accounted for. - # test on the four base-class dtypes: int, bool, float, complex + # test on the base-class dtypes: int, bool, float, complex, object # We need this test since they have no common base class. import numpy as np def find_uncallable_ufuncs(dtype): @@ -427,6 +427,7 @@ 'fabs', 'fmod', 'invert', 'mod', 'logaddexp', 'logaddexp2', 'left_shift', 'right_shift', 'copysign', 'signbit', 'ceil', 'floor', 'trunc']) + assert find_uncallable_ufuncs('object') == set() def test_int_only(self): from numpy import bitwise_and, array From noreply at buildbot.pypy.org Fri Apr 3 08:02:48 2015 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 3 Apr 2015 08:02:48 +0200 (CEST) Subject: [pypy-commit] pypy object-dtype2: add test Message-ID: <20150403060248.C09951C1331@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: object-dtype2 Changeset: r76696:7e5953936ded Date: 2015-04-03 00:23 +0300 http://bitbucket.org/pypy/pypy/changeset/7e5953936ded/ Log: add test diff --git a/pypy/module/micronumpy/test/test_object_arrays.py b/pypy/module/micronumpy/test/test_object_arrays.py --- a/pypy/module/micronumpy/test/test_object_arrays.py +++ b/pypy/module/micronumpy/test/test_object_arrays.py @@ -24,7 +24,7 @@ a = np.array(["foo"], dtype=object) b = np.array(["bar"], dtype=object) - + raises(TypeError, np.add, a, 1) res = a + b assert res[0] == "foobar" From noreply at buildbot.pypy.org Fri Apr 3 08:02:49 2015 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 3 Apr 2015 08:02:49 +0200 (CEST) Subject: [pypy-commit] pypy object-dtype2: add tests, fixes for tests Message-ID: <20150403060249.F2B141C1331@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: object-dtype2 Changeset: r76697:077446240bbd Date: 2015-04-03 09:03 +0300 http://bitbucket.org/pypy/pypy/changeset/077446240bbd/ Log: add tests, fixes for tests diff --git a/pypy/module/micronumpy/test/test_object_arrays.py b/pypy/module/micronumpy/test/test_object_arrays.py --- a/pypy/module/micronumpy/test/test_object_arrays.py +++ b/pypy/module/micronumpy/test/test_object_arrays.py @@ -28,6 +28,23 @@ res = a + b assert res[0] == "foobar" + def test_bool_func(self): + import numpy as np + a = np.array(["foo"], dtype=object) + b = a and complex(1, -1) + assert b == complex(1, -1) + b = complex(1, -1) and a + assert (b == a).all() + + def test_logical_ufunc(self): + import numpy as np + a = np.array(["foo"], dtype=object) + b = np.array([1], dtype=object) + raises(TypeError, np.logical_and, a, 1) + raises(TypeError, np.logical_and, b, complex(1, -1)) + c = b & 1 + assert (c == b).all() + def test_reduce(self): import numpy as np class O(object): diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -1697,6 +1697,11 @@ "cannot create object array/scalar from lltype") return self.BoxType(w_obj) + @specialize.argtype(1, 2) + def box_complex(self, real, imag): + w_obj = self.space.newcomplex(real, imag) + return self.BoxType(w_obj) + def str_format(self, box): return 'Object as string' #return space.str_w(space.repr(self.unbox(box))) @@ -1728,8 +1733,14 @@ def arctan2(self, v1, v2): raise oefmt(self.space.w_AttributeError, 'arctan2') - @specialize.argtype(1) + @raw_unary_op + def bool(self,v): + return not self.space.is_w(v, self.space.w_None) and \ + not self.space.eq_w(v, self.space.wrap(0)) and \ + not self.space.len_w(v) == 0 + def _bool(self, v): + #assert isinstance(v, W_Root) return self.space.bool_w(v) @raw_binary_op From noreply at buildbot.pypy.org Fri Apr 3 14:37:47 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 3 Apr 2015 14:37:47 +0200 (CEST) Subject: [pypy-commit] pypy default: Tentative fix for issue #2015. Didn't manage to write a test... Message-ID: <20150403123747.06A2D1C116D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76698:d6e8469e06ee Date: 2015-04-03 11:32 +0200 http://bitbucket.org/pypy/pypy/changeset/d6e8469e06ee/ Log: Tentative fix for issue #2015. Didn't manage to write a test... diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py --- a/rpython/jit/metainterp/optimizeopt/unroll.py +++ b/rpython/jit/metainterp/optimizeopt/unroll.py @@ -438,6 +438,9 @@ if op.is_ovf(): guard = ResOperation(rop.GUARD_NO_OVERFLOW, [], None) optimizer.send_extra_operation(guard) + if op.is_call_pure_with_exception(): + guard = ResOperation(rop.GUARD_NO_EXCEPTION, [], None) + optimizer.send_extra_operation(guard) def add_op_to_short(self, op, emit=True, guards_needed=False): if op is None: @@ -471,6 +474,9 @@ # FIXME: ensure that GUARD_OVERFLOW:ed ops not end up here guard = ResOperation(rop.GUARD_NO_OVERFLOW, [], None) self.add_op_to_short(guard, emit, guards_needed) + if op.is_call_pure_with_exception(): + guard = ResOperation(rop.GUARD_NO_EXCEPTION, [], None) + self.add_op_to_short(guard, emit, guards_needed) for guard in value_guards: self.add_op_to_short(guard, emit, guards_needed) diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -176,6 +176,12 @@ def returns_bool_result(self): return self._cls_has_bool_result + def is_call_pure_with_exception(self): + if self.getopnum() == rop.CALL_PURE: + effectinfo = self.getdescr().get_extra_info() + return effectinfo.check_can_raise() + return False + # =================== # Top of the hierachy From noreply at buildbot.pypy.org Fri Apr 3 14:37:48 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 3 Apr 2015 14:37:48 +0200 (CEST) Subject: [pypy-commit] pypy default: merge heads Message-ID: <20150403123748.5BAC01C116D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76699:25109032512b Date: 2015-04-03 14:37 +0200 http://bitbucket.org/pypy/pypy/changeset/25109032512b/ Log: merge heads diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py --- a/pypy/module/cpyext/longobject.py +++ b/pypy/module/cpyext/longobject.py @@ -186,6 +186,17 @@ pend[0] = rffi.ptradd(str, len(s)) return space.call_function(space.w_long, w_str, w_base) + at cpython_api([rffi.CWCHARP, Py_ssize_t, rffi.INT_real], PyObject) +def PyLong_FromUnicode(space, u, length, base): + """Convert a sequence of Unicode digits to a Python long integer value. + The first parameter, u, points to the first character of the Unicode + string, length gives the number of characters, and base is the radix + for the conversion. The radix must be in the range [2, 36]; if it is + out of range, ValueError will be raised.""" + w_value = space.wrap(rffi.wcharpsize2unicode(u, length)) + w_base = space.wrap(rffi.cast(lltype.Signed, base)) + return space.call_function(space.w_long, w_value, w_base) + @cpython_api([rffi.VOIDP], PyObject) def PyLong_FromVoidPtr(space, p): """Create a Python integer or long integer from the pointer p. The pointer value diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py --- a/pypy/module/cpyext/stubs.py +++ b/pypy/module/cpyext/stubs.py @@ -1395,18 +1395,6 @@ """ raise NotImplementedError - at cpython_api([rffi.CWCHARP, Py_ssize_t, rffi.INT_real], PyObject) -def PyLong_FromUnicode(space, u, length, base): - """Convert a sequence of Unicode digits to a Python long integer value. The first - parameter, u, points to the first character of the Unicode string, length - gives the number of characters, and base is the radix for the conversion. The - radix must be in the range [2, 36]; if it is out of range, ValueError - will be raised. - - This function used an int for length. This might require - changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([PyObject, rffi.CCHARP], rffi.INT_real, error=-1) def PyMapping_DelItemString(space, o, key): """Remove the mapping for object key from the object o. Return -1 on diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py --- a/pypy/module/cpyext/test/test_longobject.py +++ b/pypy/module/cpyext/test/test_longobject.py @@ -180,3 +180,16 @@ assert module.from_bytearray(False, False) == 0xBC9A assert module.from_bytearray(False, True) == -0x4365 + def test_fromunicode(self): + module = self.import_extension('foo', [ + ("from_unicode", "METH_O", + """ + Py_UNICODE* u = PyUnicode_AsUnicode(args); + return Py_BuildValue("NN", + PyLong_FromUnicode(u, 6, 10), + PyLong_FromUnicode(u, 6, 16)); + """), + ]) + # A string with arabic digits. 'BAD' is after the 6th character. + assert module.from_unicode(u' 1\u0662\u0663\u0664BAD') == (1234, 4660) + From noreply at buildbot.pypy.org Fri Apr 3 15:09:38 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 3 Apr 2015 15:09:38 +0200 (CEST) Subject: [pypy-commit] pypy default: A systematic fix for slow get_printable_location(): if we call Message-ID: <20150403130938.AE94B1C0351@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76700:c3223de868fa Date: 2015-04-03 15:09 +0200 http://bitbucket.org/pypy/pypy/changeset/c3223de868fa/ Log: A systematic fix for slow get_printable_location(): if we call get_location_str() while debug_prints are disabled, it will just return a "disabled" string now. diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py --- a/rpython/jit/metainterp/warmstate.py +++ b/rpython/jit/metainterp/warmstate.py @@ -4,6 +4,7 @@ from rpython.jit.codewriter import support, heaptracker, longlong from rpython.jit.metainterp import history from rpython.rlib.debug import debug_start, debug_stop, debug_print +from rpython.rlib.debug import have_debug_prints from rpython.rlib.jit import PARAMETERS from rpython.rlib.nonconst import NonConstant from rpython.rlib.objectmodel import specialize, we_are_translated, r_dict @@ -619,21 +620,28 @@ self.get_assembler_token = get_assembler_token # + jitdriver = self.jitdriver_sd.jitdriver + if self.jitdriver_sd.jitdriver: + drivername = jitdriver.name + else: + drivername = '' get_location_ptr = self.jitdriver_sd._get_printable_location_ptr if get_location_ptr is None: - jitdriver = self.jitdriver_sd.jitdriver - if self.jitdriver_sd.jitdriver: - drivername = jitdriver.name - else: - drivername = '' missing = '(%s: no get_printable_location)' % drivername def get_location_str(greenkey): return missing else: rtyper = self.warmrunnerdesc.rtyper unwrap_greenkey = self.make_unwrap_greenkey() + # the following missing text should not be seen, as it is + # returned only if debug_prints are currently not enabled, + # but it may show up anyway (consider it bugs) + missing = ('(%s: get_printable_location ' + 'disabled, no debug_print)' % drivername) # def get_location_str(greenkey): + if not have_debug_prints(): + return missing greenargs = unwrap_greenkey(greenkey) fn = support.maybe_on_top_of_llinterp(rtyper, get_location_ptr) llres = fn(*greenargs) From noreply at buildbot.pypy.org Fri Apr 3 17:02:07 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 3 Apr 2015 17:02:07 +0200 (CEST) Subject: [pypy-commit] pypy default: Issue #2015 second attempt: don't put 'call_pure' in short preamble, not Message-ID: <20150403150207.D6D691C116D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76701:56eebe9dd813 Date: 2015-04-03 15:28 +0200 http://bitbucket.org/pypy/pypy/changeset/56eebe9dd813/ Log: Issue #2015 second attempt: don't put 'call_pure' in short preamble, not at all, if they can raise. I *think* the issue is that an exception that we'd get then would be propagated outside, uncaught. diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py --- a/rpython/jit/metainterp/optimizeopt/pure.py +++ b/rpython/jit/metainterp/optimizeopt/pure.py @@ -80,7 +80,13 @@ args = op.getarglist() self.emit_operation(ResOperation(rop.CALL, args, op.result, op.getdescr())) - self.call_pure_positions.append(len(self.optimizer._newoperations) - 1) + + # don't move call_pure_with_exception in the short preamble... + # issue #2015 + effectinfo = op.getdescr().get_extra_info() + if not effectinfo.check_can_raise(): + self.call_pure_positions.append( + len(self.optimizer._newoperations) - 1) def optimize_GUARD_NO_EXCEPTION(self, op): if self.last_emitted_operation is REMOVED: diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -3605,7 +3605,7 @@ ops = ''' [p1, i1, i4] setfield_gc(p1, i1, descr=valuedescr) - i3 = call_pure(p1, descr=plaincalldescr) + i3 = call_pure(p1, descr=cannotraisecalldescr) setfield_gc(p1, i3, descr=valuedescr) jump(p1, i4, i3) ''' @@ -3617,7 +3617,7 @@ preamble = ''' [p1, i1, i4] setfield_gc(p1, i1, descr=valuedescr) - i3 = call(p1, descr=plaincalldescr) + i3 = call(p1, descr=cannotraisecalldescr) setfield_gc(p1, i3, descr=valuedescr) i148 = same_as(i3) i147 = same_as(i3) @@ -3630,7 +3630,7 @@ ops = ''' [p1, i1, i4] setfield_gc(p1, i1, descr=valuedescr) - i3 = call_pure(p1, descr=plaincalldescr) + i3 = call_pure(p1, descr=cannotraisecalldescr) setfield_gc(p1, i1, descr=valuedescr) jump(p1, i4, i3) ''' @@ -3642,7 +3642,7 @@ preamble = ''' [p1, i1, i4] setfield_gc(p1, i1, descr=valuedescr) - i3 = call(p1, descr=plaincalldescr) + i3 = call(p1, descr=cannotraisecalldescr) setfield_gc(p1, i1, descr=valuedescr) i151 = same_as(i3) jump(p1, i4, i3, i151) @@ -3656,15 +3656,15 @@ [i0, i1, i2] escape(i1) escape(i2) - i3 = call_pure(123456, 4, 5, 6, descr=plaincalldescr) - i4 = call_pure(123456, 4, i0, 6, descr=plaincalldescr) + i3 = call_pure(123456, 4, 5, 6, descr=cannotraisecalldescr) + i4 = call_pure(123456, 4, i0, 6, descr=cannotraisecalldescr) jump(i0, i3, i4) ''' preamble = ''' [i0, i1, i2] escape(i1) escape(i2) - i4 = call(123456, 4, i0, 6, descr=plaincalldescr) + i4 = call(123456, 4, i0, 6, descr=cannotraisecalldescr) i153 = same_as(i4) jump(i0, i4, i153) ''' @@ -3678,6 +3678,8 @@ def test_call_pure_constant_folding_exc(self): # CALL_PURE may be followed by GUARD_NO_EXCEPTION + # XXX maybe temporary, but we can't remove such call_pures from + # the loop, because the short preamble can't call them safely. arg_consts = [ConstInt(i) for i in (123456, 4, 5, 6)] call_pure_results = {tuple(arg_consts): ConstInt(42)} ops = ''' @@ -3696,14 +3698,15 @@ escape(i2) i4 = call(123456, 4, i0, 6, descr=plaincalldescr) guard_no_exception() [] - i155 = same_as(i4) - jump(i0, i4, i155) + jump(i0, i4) ''' expected = ''' - [i0, i2, i3] + [i0, i2] escape(42) escape(i2) - jump(i0, i3, i3) + i4 = call(123456, 4, i0, 6, descr=plaincalldescr) + guard_no_exception() [] + jump(i0, i4) ''' self.optimize_loop(ops, expected, preamble, call_pure_results) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py --- a/rpython/jit/metainterp/optimizeopt/test/test_util.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py @@ -182,6 +182,9 @@ FUNC = lltype.FuncType([lltype.Signed], lltype.Signed) plaincalldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo.MOST_GENERAL) + cannotraisecalldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, + EffectInfo([], [], [], [valuedescr], [], [], + EffectInfo.EF_CANNOT_RAISE)) nonwritedescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo([], [], [], [], [], [])) writeadescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py --- a/rpython/jit/metainterp/optimizeopt/unroll.py +++ b/rpython/jit/metainterp/optimizeopt/unroll.py @@ -438,9 +438,7 @@ if op.is_ovf(): guard = ResOperation(rop.GUARD_NO_OVERFLOW, [], None) optimizer.send_extra_operation(guard) - if op.is_call_pure_with_exception(): - guard = ResOperation(rop.GUARD_NO_EXCEPTION, [], None) - optimizer.send_extra_operation(guard) + assert not op.is_call_pure_with_exception() def add_op_to_short(self, op, emit=True, guards_needed=False): if op is None: @@ -474,9 +472,7 @@ # FIXME: ensure that GUARD_OVERFLOW:ed ops not end up here guard = ResOperation(rop.GUARD_NO_OVERFLOW, [], None) self.add_op_to_short(guard, emit, guards_needed) - if op.is_call_pure_with_exception(): - guard = ResOperation(rop.GUARD_NO_EXCEPTION, [], None) - self.add_op_to_short(guard, emit, guards_needed) + assert not op.is_call_pure_with_exception() for guard in value_guards: self.add_op_to_short(guard, emit, guards_needed) From noreply at buildbot.pypy.org Fri Apr 3 17:02:09 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 3 Apr 2015 17:02:09 +0200 (CEST) Subject: [pypy-commit] pypy default: 'fix' test Message-ID: <20150403150209.160771C116D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76702:b58c996fdef5 Date: 2015-04-03 16:15 +0200 http://bitbucket.org/pypy/pypy/changeset/b58c996fdef5/ Log: 'fix' test diff --git a/rpython/jit/metainterp/test/test_string.py b/rpython/jit/metainterp/test/test_string.py --- a/rpython/jit/metainterp/test/test_string.py +++ b/rpython/jit/metainterp/test/test_string.py @@ -897,7 +897,9 @@ m -= 1 return 42 self.meta_interp(f, [6, 7]) - self.check_resops(unicodesetitem=2, newunicode=2, call=4, + # xxx used to be 'call=4', but the two extra calls in the loop + # are not safe to remove; see 56eebe9dd813 + self.check_resops(unicodesetitem=2, newunicode=2, call=6, copyunicodecontent=2, unicodegetitem=0) def test_str2unicode_fold(self): From noreply at buildbot.pypy.org Fri Apr 3 19:06:58 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 3 Apr 2015 19:06:58 +0200 (CEST) Subject: [pypy-commit] pypy vmprof: Kill "recursive", with comment Message-ID: <20150403170658.92A421C050C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof Changeset: r76703:8ef1c41ed611 Date: 2015-04-03 19:06 +0200 http://bitbucket.org/pypy/pypy/changeset/8ef1c41ed611/ Log: Kill "recursive", with comment diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c --- a/pypy/module/_vmprof/src/vmprof.c +++ b/pypy/module/_vmprof/src/vmprof.c @@ -140,29 +140,25 @@ * ************************************************************* */ -// stolen from pprof: -// Sometimes, we can try to get a stack trace from within a stack -// trace, because libunwind can call mmap (maybe indirectly via an -// internal mmap based memory allocator), and that mmap gets trapped -// and causes a stack-trace request. If were to try to honor that -// recursive request, we'd end up with infinite recursion or deadlock. -// Luckily, it's safe to ignore those subsequent traces. In such -// cases, we return 0 to indicate the situation. +// The original code here has a comment, "stolen from pprof", +// about a "__thread int recursive". But general __thread +// variables are not really supposed to be accessed from a +// signal handler. Moreover, we are using SIGPROF, which +// should not be recursively called on the same thread. //static __thread int recursive; -static int recursive; // XXX antocuni: removed __thread int get_stack_trace(void** result, int max_depth, ucontext_t *ucontext) { void *ip; int n = 0; unw_cursor_t cursor; unw_context_t uc = *ucontext; - if (recursive) { + //if (recursive) { + // return 0; + //} + if (!custom_sanity_check()) { return 0; } - if (!custom_sanity_check()) { - return 0; - } - ++recursive; + //++recursive; int ret = unw_init_local(&cursor, &uc); assert(ret >= 0); @@ -209,7 +205,7 @@ } first_run = 0; } - --recursive; + //--recursive; return n; } From noreply at buildbot.pypy.org Fri Apr 3 19:17:07 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 3 Apr 2015 19:17:07 +0200 (CEST) Subject: [pypy-commit] pypy vmprof: API change: _vmprof.enable() second argument is now measured in seconds, Message-ID: <20150403171707.2289C1C116D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof Changeset: r76704:6fd836a1af7b Date: 2015-04-03 19:17 +0200 http://bitbucket.org/pypy/pypy/changeset/6fd836a1af7b/ Log: API change: _vmprof.enable() second argument is now measured in seconds, given as a float, as customary in Python diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py --- a/pypy/module/_vmprof/interp_vmprof.py +++ b/pypy/module/_vmprof/interp_vmprof.py @@ -134,12 +134,12 @@ self.fileno = -1 self.current_codes = [] - def enable(self, space, fileno, period): + def enable(self, space, fileno, period_usec): if self.is_enabled: raise oefmt(space.w_ValueError, "_vmprof already enabled") self.fileno = fileno self.is_enabled = True - self.write_header(fileno, period) + self.write_header(fileno, period_usec) if not self.ever_enabled: if we_are_translated(): pypy_vmprof_init() @@ -148,7 +148,7 @@ space.register_code_callback(vmprof_register_code) if we_are_translated(): # does not work untranslated - res = vmprof_enable(fileno, period, 0, + res = vmprof_enable(fileno, period_usec, 0, lltype.nullptr(rffi.CCHARP.TO), 0) else: res = 0 @@ -161,11 +161,8 @@ for code in all_code_objs: self.register_code(space, code) - def write_header(self, fileno, period): - if period == -1: - period_usec = 1000000 / 100 # 100hz - else: - period_usec = period + def write_header(self, fileno, period_usec): + assert period_usec > 0 b = StringBuilder() write_long_to_string_builder(0, b) write_long_to_string_builder(3, b) @@ -217,13 +214,22 @@ mod_vmprof = space.getbuiltinmodule('_vmprof') assert isinstance(mod_vmprof, Module) mod_vmprof.vmprof.register_code(space, code) - - at unwrap_spec(fileno=int, period=int) -def enable(space, fileno, period=-1): + + at unwrap_spec(fileno=int, period=float) +def enable(space, fileno, period=0.01): # default 100 Hz from pypy.module._vmprof import Module mod_vmprof = space.getbuiltinmodule('_vmprof') assert isinstance(mod_vmprof, Module) - mod_vmprof.vmprof.enable(space, fileno, period) + # + try: + period_usec = int(period * 1000000.0 + 0.5) + if period_usec <= 0: + raise ValueError + except (ValueError, OverflowError): + raise OperationError(self.w_ValueError, + self.wrap("'period' too large or non positive")) + # + mod_vmprof.vmprof.enable(space, fileno, period_usec) def disable(space): from pypy.module._vmprof import Module diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c --- a/pypy/module/_vmprof/src/vmprof.c +++ b/pypy/module/_vmprof/src/vmprof.c @@ -65,6 +65,7 @@ } static void prof_header(long period_usec) { + // XXX never used here? prof_word(0); prof_word(3); prof_word(0); @@ -351,8 +352,7 @@ int vmprof_enable(int fd, long period_usec, int write_header, char *s, int slen) { - if (period_usec == -1) - period_usec = 1000000 / 100; /* 100hz */ + assert(period_usec > 0); if (open_profile(fd, period_usec, write_header, s, slen) == -1) { return -1; } From noreply at buildbot.pypy.org Fri Apr 3 19:18:30 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 3 Apr 2015 19:18:30 +0200 (CEST) Subject: [pypy-commit] pypy vmprof: API change Message-ID: <20150403171830.92DB41C116D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof Changeset: r76705:cd020116336a Date: 2015-04-03 19:18 +0200 http://bitbucket.org/pypy/pypy/changeset/cd020116336a/ Log: API change diff --git a/rpython/bin/rpython-vmprof b/rpython/bin/rpython-vmprof --- a/rpython/bin/rpython-vmprof +++ b/rpython/bin/rpython-vmprof @@ -19,7 +19,7 @@ import _vmprof, subprocess x = subprocess.Popen('gzip > vmprof.log.gz', shell=True, stdin=subprocess.PIPE) -_vmprof.enable(x.stdin.fileno(), 1000) +_vmprof.enable(x.stdin.fileno(), 0.001) try: main() finally: From noreply at buildbot.pypy.org Fri Apr 3 19:47:55 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 3 Apr 2015 19:47:55 +0200 (CEST) Subject: [pypy-commit] pypy.org extradoc: update the values Message-ID: <20150403174755.920411C050C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r593:90cced881ed2 Date: 2015-04-03 19:48 +0200 http://bitbucket.org/pypy/pypy.org/changeset/90cced881ed2/ Log: update the values diff --git a/don1.html b/don1.html --- a/don1.html +++ b/don1.html @@ -15,7 +15,7 @@ - $59106 of $105000 (56.3%) + $59130 of $105000 (56.3%)

diff --git a/don4.html b/don4.html --- a/don4.html +++ b/don4.html @@ -17,7 +17,7 @@ 2nd call: - $22663 of $80000 (28.3%) + $22674 of $80000 (28.3%)

From noreply at buildbot.pypy.org Sat Apr 4 10:58:21 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 4 Apr 2015 10:58:21 +0200 (CEST) Subject: [pypy-commit] pypy vmprof: Another failing test Message-ID: <20150404085821.5B1261C0351@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof Changeset: r76706:3daf0e8619bc Date: 2015-04-04 10:58 +0200 http://bitbucket.org/pypy/pypy/changeset/3daf0e8619bc/ Log: Another failing test diff --git a/rpython/jit/backend/llsupport/test/test_codemap.py b/rpython/jit/backend/llsupport/test/test_codemap.py --- a/rpython/jit/backend/llsupport/test/test_codemap.py +++ b/rpython/jit/backend/llsupport/test/test_codemap.py @@ -91,3 +91,8 @@ codemap.free_asm_block(200, 300) assert unpack_traceback(225) == [] codemap.free() + +def test_codemaps_bug1(): + builder = CodemapBuilder() + builder.debug_merge_point(3, 102, 0) + builder.debug_merge_point(0, 104, 10) From noreply at buildbot.pypy.org Sat Apr 4 11:11:35 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 4 Apr 2015 11:11:35 +0200 (CEST) Subject: [pypy-commit] pypy default: Fix for test_sqlite3 Message-ID: <20150404091135.73C1F1C0351@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76707:7c75c5d7b8db Date: 2015-04-04 11:11 +0200 http://bitbucket.org/pypy/pypy/changeset/7c75c5d7b8db/ Log: Fix for test_sqlite3 diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -1246,7 +1246,6 @@ ret = _lib.sqlite3_prepare_v2(self.__con._db, c_sql, -1, statement_star, next_char) self._statement = statement_star[0] - self.__con._remember_statement(self) if ret == _lib.SQLITE_OK and not self._statement: # an empty statement, work around that, as it's the least trouble @@ -1259,6 +1258,8 @@ if ret != _lib.SQLITE_OK: raise self.__con._get_exception(ret) + self.__con._remember_statement(self) + tail = _ffi.string(next_char[0]).decode('utf-8') if _check_remaining_sql(tail): raise Warning("You can only execute one statement at a time.") From noreply at buildbot.pypy.org Sat Apr 4 11:18:54 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 4 Apr 2015 11:18:54 +0200 (CEST) Subject: [pypy-commit] pypy default: Ask for get_location_str() within the "jit-backend" debug_start/debug_stop Message-ID: <20150404091854.F0E931C0351@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76708:91bb1d0abd09 Date: 2015-04-04 11:19 +0200 http://bitbucket.org/pypy/pypy/changeset/91bb1d0abd09/ Log: Ask for get_location_str() within the "jit-backend" debug_start/debug_stop diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -325,7 +325,6 @@ patch_new_loop_to_load_virtualizable_fields(loop, jitdriver_sd) original_jitcell_token = loop.original_jitcell_token - loopname = jitdriver_sd.warmstate.get_location_str(greenkey) globaldata = metainterp_sd.globaldata original_jitcell_token.number = n = globaldata.loopnumbering globaldata.loopnumbering += 1 @@ -347,6 +346,7 @@ metainterp_sd.profiler.start_backend() debug_start("jit-backend") try: + loopname = jitdriver_sd.warmstate.get_location_str(greenkey) asminfo = do_compile_loop(metainterp_sd, loop.inputargs, operations, original_jitcell_token, name=loopname) From noreply at buildbot.pypy.org Sat Apr 4 11:57:18 2015 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 4 Apr 2015 11:57:18 +0200 (CEST) Subject: [pypy-commit] pypy vmprof: fix codemap.c (passes tests now) Message-ID: <20150404095718.EC11C1C1CD4@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76709:c5dd2595fc47 Date: 2015-04-04 11:57 +0200 http://bitbucket.org/pypy/pypy/changeset/c5dd2595fc47/ Log: fix codemap.c (passes tests now) diff --git a/rpython/jit/backend/llsupport/src/codemap.c b/rpython/jit/backend/llsupport/src/codemap.c --- a/rpython/jit/backend/llsupport/src/codemap.c +++ b/rpython/jit/backend/llsupport/src/codemap.c @@ -82,17 +82,19 @@ codemap_data_t *data; uintptr_t rel_addr; - if (codemap == &jit_codemap_head) + if (codemap == &jit_codemap_head) { if (start_addr) *start_addr = 0; return NULL; + } rel_addr = (uintptr_t)addr - codemap->key; data = (codemap_data_t *)codemap->data; - if (rel_addr >= data->machine_code_size) + if (rel_addr >= data->machine_code_size) { if (start_addr) *start_addr = 0; return NULL; + } if (start_addr) *start_addr = (long)codemap->key; From noreply at buildbot.pypy.org Sat Apr 4 12:11:50 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 4 Apr 2015 12:11:50 +0200 (CEST) Subject: [pypy-commit] pypy vmprof: The new test should never occur in practice. Add an assert and Message-ID: <20150404101150.1AF961C0351@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof Changeset: r76710:c6efe755d7b3 Date: 2015-04-04 12:11 +0200 http://bitbucket.org/pypy/pypy/changeset/c6efe755d7b3/ Log: The new test should never occur in practice. Add an assert and an explanation. diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py --- a/rpython/jit/backend/llsupport/codemap.py +++ b/rpython/jit/backend/llsupport/codemap.py @@ -151,6 +151,12 @@ return assert unique_id & 1 == 0 if call_depth > self.last_call_depth: + assert call_depth == self.last_call_depth + 1 + # ^^^ It should never be the case that we see + # debug_merge_points that suddenly go more than *one* + # call deeper than the previous one (unless we're at + # the start of a bridge, handled by + # inherit_code_from_position()). self.l.append(unique_id) self.l.append(pos) # <- this is a relative pos self.patch_position.append(len(self.l)) diff --git a/rpython/jit/backend/llsupport/test/test_codemap.py b/rpython/jit/backend/llsupport/test/test_codemap.py --- a/rpython/jit/backend/llsupport/test/test_codemap.py +++ b/rpython/jit/backend/llsupport/test/test_codemap.py @@ -91,8 +91,3 @@ codemap.free_asm_block(200, 300) assert unpack_traceback(225) == [] codemap.free() - -def test_codemaps_bug1(): - builder = CodemapBuilder() - builder.debug_merge_point(3, 102, 0) - builder.debug_merge_point(0, 104, 10) From noreply at buildbot.pypy.org Sat Apr 4 12:21:53 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 4 Apr 2015 12:21:53 +0200 (CEST) Subject: [pypy-commit] pypy vmprof: Test and fix Message-ID: <20150404102153.2EDAD1C0351@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof Changeset: r76711:e492b2d111b0 Date: 2015-04-04 12:22 +0200 http://bitbucket.org/pypy/pypy/changeset/e492b2d111b0/ Log: Test and fix diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py --- a/pypy/module/_vmprof/interp_vmprof.py +++ b/pypy/module/_vmprof/interp_vmprof.py @@ -4,6 +4,7 @@ from rpython.rtyper.annlowlevel import cast_instance_to_gcref, cast_base_ptr_to_instance from rpython.rlib.objectmodel import we_are_translated from rpython.rlib import jit, rposix, rgc +from rpython.rlib.rarithmetic import ovfcheck_float_to_int from rpython.rtyper.tool import rffi_platform as platform from rpython.rlib.rstring import StringBuilder from pypy.interpreter.baseobjspace import W_Root @@ -222,12 +223,12 @@ assert isinstance(mod_vmprof, Module) # try: - period_usec = int(period * 1000000.0 + 0.5) + period_usec = ovfcheck_float_to_int(period * 1000000.0 + 0.5) if period_usec <= 0: raise ValueError except (ValueError, OverflowError): - raise OperationError(self.w_ValueError, - self.wrap("'period' too large or non positive")) + raise OperationError(space.w_ValueError, + space.wrap("'period' too large or non positive")) # mod_vmprof.vmprof.enable(space, fileno, period_usec) diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py --- a/pypy/module/_vmprof/test/test__vmprof.py +++ b/pypy/module/_vmprof/test/test__vmprof.py @@ -58,3 +58,11 @@ assert "py:foo:" in s assert "py:foo2:" in s assert no_of_codes2 >= no_of_codes + 2 # some extra codes from tests + + def test_enable_ovf(self): + import _vmprof + raises(ValueError, _vmprof.enable, 999, 0) + raises(ValueError, _vmprof.enable, 999, -2.5) + raises(ValueError, _vmprof.enable, 999, 1e300) + raises(ValueError, _vmprof.enable, 999, 1e300 * 1e300) + raises(ValueError, _vmprof.enable, 999, (1e300*1e300) / (1e300*1e300)) From noreply at buildbot.pypy.org Sat Apr 4 13:32:59 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 4 Apr 2015 13:32:59 +0200 (CEST) Subject: [pypy-commit] pypy default: Add a comment Message-ID: <20150404113259.1AD6F1C0A5B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76712:84e6b9734fbf Date: 2015-04-04 13:32 +0200 http://bitbucket.org/pypy/pypy/changeset/84e6b9734fbf/ Log: Add a comment diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -1176,6 +1176,10 @@ if have_debug_prints(): loc = jitdriver_sd.warmstate.get_location_str(greenkey) debug_print(loc) + # + # Note: the logger hides the jd_index argument, so we see in the logs: + # debug_merge_point(portal_call_depth, current_call_id, 'location') + # args = [ConstInt(jd_index), ConstInt(portal_call_depth), ConstInt(current_call_id)] + greenkey self.metainterp.history.record(rop.DEBUG_MERGE_POINT, args, None) From noreply at buildbot.pypy.org Sat Apr 4 18:33:13 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 4 Apr 2015 18:33:13 +0200 (CEST) Subject: [pypy-commit] pypy default: Issue #2017: trying to limit the virtual address space consumption Message-ID: <20150404163313.588971C0351@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76713:ef621d60ea13 Date: 2015-04-04 18:32 +0200 http://bitbucket.org/pypy/pypy/changeset/ef621d60ea13/ Log: Issue #2017: trying to limit the virtual address space consumption of having tons of greenlets. diff --git a/rpython/memory/gctransform/shadowstack.py b/rpython/memory/gctransform/shadowstack.py --- a/rpython/memory/gctransform/shadowstack.py +++ b/rpython/memory/gctransform/shadowstack.py @@ -270,15 +270,16 @@ class ShadowStackPool(object): """Manages a pool of shadowstacks. The MAX most recently used - shadowstacks are fully allocated and can be directly jumped into. + shadowstacks are fully allocated and can be directly jumped into + (called "full stacks" below). The rest are stored in a more virtual-memory-friendly way, i.e. with just the right amount malloced. Before they can run, they - must be copied into a full shadowstack. XXX NOT IMPLEMENTED SO FAR! + must be copied into a full shadowstack. """ _alloc_flavor_ = "raw" root_stack_depth = 163840 - #MAX = 20 not implemented yet + MAX = 20 def __init__(self, gcdata): self.unused_full_stack = llmemory.NULL @@ -299,13 +300,21 @@ forget_current_state(), and then call restore_state_from() or start_fresh_new_state(). """ - self._prepare_unused_stack() + fresh_free_fullstack = shadowstackref.prepare_free_slot() + if self.unused_full_stack: + if fresh_free_fullstack: + llmemory.raw_free(fresh_free_fullstack) + elif fresh_free_fullstack: + self.unused_full_stack = fresh_free_fullstack + else: + self._prepare_unused_stack() + # shadowstackref.base = self.gcdata.root_stack_base shadowstackref.top = self.gcdata.root_stack_top shadowstackref.context = ncontext ll_assert(shadowstackref.base <= shadowstackref.top, "save_current_state_away: broken shadowstack") - #shadowstackref.fullstack = True + shadowstackref.attach() # # cannot use llop.gc_writebarrier() here, because # we are in a minimally-transformed GC helper :-/ @@ -328,6 +337,7 @@ ll_assert(bool(shadowstackref.base), "empty shadowstackref!") ll_assert(shadowstackref.base <= shadowstackref.top, "restore_state_from: broken shadowstack") + self.unused_full_stack = shadowstackref.rebuild(self.unused_full_stack) self.gcdata.root_stack_base = shadowstackref.base self.gcdata.root_stack_top = shadowstackref.top self._cleanup(shadowstackref) @@ -343,26 +353,115 @@ shadowstackref.context = llmemory.NULL def _prepare_unused_stack(self): + ll_assert(self.unused_full_stack == llmemory.NULL, + "already an unused_full_stack") + root_stack_size = sizeofaddr * self.root_stack_depth + self.unused_full_stack = llmemory.raw_malloc(root_stack_size) if self.unused_full_stack == llmemory.NULL: - root_stack_size = sizeofaddr * self.root_stack_depth - self.unused_full_stack = llmemory.raw_malloc(root_stack_size) - if self.unused_full_stack == llmemory.NULL: - raise MemoryError + raise MemoryError def get_shadowstackref(root_walker, gctransformer): if hasattr(gctransformer, '_SHADOWSTACKREF'): return gctransformer._SHADOWSTACKREF + # Helpers to same virtual address space by limiting to MAX the + # number of full shadow stacks. If there are more, we compact + # them into a separately-allocated zone of memory of just the right + # size. See the comments in the definition of fullstack_cache below. + + def ll_prepare_free_slot(_unused): + """Free up a slot in the array of MAX entries, ready for storing + a new shadowstackref. Return the memory of the now-unused full + shadowstack. + """ + index = fullstack_cache[0] + if index > 0: + return llmemory.NULL # there is already at least one free slot + # + # make a compact copy in one old entry and return the + # original full-sized memory + index = -index + ll_assert(index > 0, "prepare_free_slot: cache[0] == 0") + compacting = lltype.cast_int_to_ptr(SHADOWSTACKREFPTR, + fullstack_cache[index]) + index += 1 + if index >= ShadowStackPool.MAX: + index = 1 + fullstack_cache[0] = -index # update to the next value in order + # + compacting.detach() + original = compacting.base + size = compacting.top - original + new = llmemory.raw_malloc(size) + if new == llmemory.NULL: + return llmemory.NULL + llmemory.raw_memcopy(original, new, size) + compacting.base = new + compacting.top = new + size + return original + + def ll_attach(shadowstackref): + """After prepare_free_slot(), store a shadowstackref in that slot.""" + index = fullstack_cache[0] + ll_assert(index > 0, "fullstack attach: no free slot") + fullstack_cache[0] = fullstack_cache[index] + fullstack_cache[index] = lltype.cast_ptr_to_int(shadowstackref) + ll_assert(shadowstackref.fsindex == 0, "fullstack attach: already one?") + shadowstackref.fsindex = index # > 0 + + def ll_detach(shadowstackref): + """Detach a shadowstackref from the array of MAX entries.""" + index = shadowstackref.fsindex + ll_assert(index > 0, "detach: unattached shadowstackref") + ll_assert(fullstack_cache[index] == + lltype.cast_ptr_to_int(shadowstackref), + "detach: bad fullstack_cache") + shadowstackref.fsindex = 0 + fullstack_cache[index] = fullstack_cache[0] + fullstack_cache[0] = index + + def ll_rebuild(shadowstackref, fullstack_base): + if shadowstackref.fsindex > 0: + shadowstackref.detach() + return fullstack_base + else: + # make an expanded copy of the compact shadowstack stored in + # 'shadowstackref' and free that + compact = shadowstackref.base + size = shadowstackref.top - compact + shadowstackref.base = fullstack_base + shadowstackref.top = fullstack_base + size + llmemory.raw_memcopy(compact, fullstack_base, size) + llmemory.raw_free(compact) + return llmemory.NULL + SHADOWSTACKREFPTR = lltype.Ptr(lltype.GcForwardReference()) SHADOWSTACKREF = lltype.GcStruct('ShadowStackRef', - ('base', llmemory.Address), - ('top', llmemory.Address), - ('context', llmemory.Address), - #('fullstack', lltype.Bool), - rtti=True) + ('base', llmemory.Address), + ('top', llmemory.Address), + ('context', llmemory.Address), + ('fsindex', lltype.Signed), + rtti=True, + adtmeths={'prepare_free_slot': ll_prepare_free_slot, + 'attach': ll_attach, + 'detach': ll_detach, + 'rebuild': ll_rebuild}) SHADOWSTACKREFPTR.TO.become(SHADOWSTACKREF) + # Items 1..MAX-1 of the following array can be SHADOWSTACKREF + # addresses cast to integer. Or, they are small numbers and they + # make up a free list, rooted in item 0, which goes on until + # terminated with a negative item. This negative item gives (the + # opposite of) the index of the entry we try to remove next. + # Initially all items are in this free list and the end is '-1'. + fullstack_cache = lltype.malloc(lltype.Array(lltype.Signed), + ShadowStackPool.MAX, + flavor='raw', immortal=True) + for i in range(len(fullstack_cache) - 1): + fullstack_cache[i] = i + 1 + fullstack_cache[len(fullstack_cache) - 1] = -1 + def customtrace(gc, obj, callback, arg): obj = llmemory.cast_adr_to_ptr(obj, SHADOWSTACKREFPTR) addr = obj.top @@ -384,6 +483,8 @@ h = llmemory.cast_adr_to_ptr(h, _c.handle) shadowstackref.context = llmemory.NULL # + if shadowstackref.fsindex > 0: + shadowstackref.detach() base = shadowstackref.base shadowstackref.base = llmemory.NULL shadowstackref.top = llmemory.NULL From noreply at buildbot.pypy.org Sat Apr 4 19:41:57 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 4 Apr 2015 19:41:57 +0200 (CEST) Subject: [pypy-commit] pypy default: Document ef621d60ea13. Message-ID: <20150404174157.4C0FE1C0351@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76714:d8bbbc39f7ad Date: 2015-04-04 19:42 +0200 http://bitbucket.org/pypy/pypy/changeset/d8bbbc39f7ad/ Log: Document ef621d60ea13. diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,6 +5,11 @@ .. this is a revision shortly after release-2.5.1 .. startrev: cb01edcb59414d9d93056e54ed060673d24e67c1 +Issue #2017: on non-Linux-x86 platforms, reduced the memory impact of +creating a lot of greenlets/tasklets. Particularly useful on Win32 and +on ARM, where you used to get a MemoryError after only 2500-5000 +greenlets (the 32-bit address space is exhausted). + .. branch: gc-incminimark-pinning-improve Object Pinning is now used in `bz2` and `rzlib` (therefore also affects Python's `zlib`). In case the data to compress/decompress is inside the nursery From noreply at buildbot.pypy.org Sun Apr 5 09:17:32 2015 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 5 Apr 2015 09:17:32 +0200 (CEST) Subject: [pypy-commit] pypy vmprof: swap the order of codemap entries Message-ID: <20150405071732.5B7B21C0334@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76715:5275e224f8c8 Date: 2015-04-05 09:16 +0200 http://bitbucket.org/pypy/pypy/changeset/5275e224f8c8/ Log: swap the order of codemap entries diff --git a/pypy/module/_vmprof/src/get_custom_offset.c b/pypy/module/_vmprof/src/get_custom_offset.c --- a/pypy/module/_vmprof/src/get_custom_offset.c +++ b/pypy/module/_vmprof/src/get_custom_offset.c @@ -29,6 +29,8 @@ intptr_t id; long start_addr = 0; intptr_t addr = (intptr_t)ip; + int start, k; + void *tmp; codemap = pypy_find_codemap_at_addr(addr, &start_addr); if (codemap == NULL) @@ -38,6 +40,7 @@ // modify the last entry to point to start address and not the random one // in the middle result[n - 1] = (void*)start_addr; + start = n; while (n < max_depth) { id = pypy_yield_codemap_at_addr(codemap, addr, ¤t_pos); if (id == 0) @@ -45,5 +48,12 @@ break; result[n++] = (void *)id; } + k = 0; + while (k < (n - start) / 2) { + tmp = result[start + k]; + result[start + k] = result[n - k - 1]; + result[n - k - 1] = tmp; + k++; + } return n; } From noreply at buildbot.pypy.org Sun Apr 5 09:17:33 2015 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 5 Apr 2015 09:17:33 +0200 (CEST) Subject: [pypy-commit] pypy vmprof: merge Message-ID: <20150405071733.907821C0334@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76716:0e77fb70def9 Date: 2015-04-05 09:17 +0200 http://bitbucket.org/pypy/pypy/changeset/0e77fb70def9/ Log: merge diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py --- a/pypy/module/_vmprof/interp_vmprof.py +++ b/pypy/module/_vmprof/interp_vmprof.py @@ -4,6 +4,7 @@ from rpython.rtyper.annlowlevel import cast_instance_to_gcref, cast_base_ptr_to_instance from rpython.rlib.objectmodel import we_are_translated from rpython.rlib import jit, rposix, rgc +from rpython.rlib.rarithmetic import ovfcheck_float_to_int from rpython.rtyper.tool import rffi_platform as platform from rpython.rlib.rstring import StringBuilder from pypy.interpreter.baseobjspace import W_Root @@ -222,12 +223,12 @@ assert isinstance(mod_vmprof, Module) # try: - period_usec = int(period * 1000000.0 + 0.5) + period_usec = ovfcheck_float_to_int(period * 1000000.0 + 0.5) if period_usec <= 0: raise ValueError except (ValueError, OverflowError): - raise OperationError(self.w_ValueError, - self.wrap("'period' too large or non positive")) + raise OperationError(space.w_ValueError, + space.wrap("'period' too large or non positive")) # mod_vmprof.vmprof.enable(space, fileno, period_usec) diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py --- a/pypy/module/_vmprof/test/test__vmprof.py +++ b/pypy/module/_vmprof/test/test__vmprof.py @@ -58,3 +58,11 @@ assert "py:foo:" in s assert "py:foo2:" in s assert no_of_codes2 >= no_of_codes + 2 # some extra codes from tests + + def test_enable_ovf(self): + import _vmprof + raises(ValueError, _vmprof.enable, 999, 0) + raises(ValueError, _vmprof.enable, 999, -2.5) + raises(ValueError, _vmprof.enable, 999, 1e300) + raises(ValueError, _vmprof.enable, 999, 1e300 * 1e300) + raises(ValueError, _vmprof.enable, 999, (1e300*1e300) / (1e300*1e300)) diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py --- a/rpython/jit/backend/llsupport/codemap.py +++ b/rpython/jit/backend/llsupport/codemap.py @@ -151,6 +151,12 @@ return assert unique_id & 1 == 0 if call_depth > self.last_call_depth: + assert call_depth == self.last_call_depth + 1 + # ^^^ It should never be the case that we see + # debug_merge_points that suddenly go more than *one* + # call deeper than the previous one (unless we're at + # the start of a bridge, handled by + # inherit_code_from_position()). self.l.append(unique_id) self.l.append(pos) # <- this is a relative pos self.patch_position.append(len(self.l)) diff --git a/rpython/jit/backend/llsupport/test/test_codemap.py b/rpython/jit/backend/llsupport/test/test_codemap.py --- a/rpython/jit/backend/llsupport/test/test_codemap.py +++ b/rpython/jit/backend/llsupport/test/test_codemap.py @@ -91,8 +91,3 @@ codemap.free_asm_block(200, 300) assert unpack_traceback(225) == [] codemap.free() - -def test_codemaps_bug1(): - builder = CodemapBuilder() - builder.debug_merge_point(3, 102, 0) - builder.debug_merge_point(0, 104, 10) From noreply at buildbot.pypy.org Sun Apr 5 12:03:23 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 5 Apr 2015 12:03:23 +0200 (CEST) Subject: [pypy-commit] pypy default: remove unmaintained files of no use to us (as far as I know) Message-ID: <20150405100323.AA36D1C0204@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76717:b506da49b72e Date: 2015-04-05 12:03 +0200 http://bitbucket.org/pypy/pypy/changeset/b506da49b72e/ Log: remove unmaintained files of no use to us (as far as I know) diff --git a/.tddium.requirements.txt b/.tddium.requirements.txt deleted file mode 100644 --- a/.tddium.requirements.txt +++ /dev/null @@ -1,1 +0,0 @@ -pytest diff --git a/tddium.yml b/tddium.yml deleted file mode 100644 --- a/tddium.yml +++ /dev/null @@ -1,18 +0,0 @@ -tddium: - timeout: 1800 - python: - python_version: 2.7 - pip_requirements_file: .tddium.requirements.txt # will go away soon - tests: - - type: pytest - mode: parallel - files: - - pypy/**/test_*.py - - rpython/**/test_*.py - - exclude: pypy/module/test_lib_pypy/ctypes_tests/** # don't run in CPython - - exclude: rpython/jit/backend/llvm/** # bitrotted AFAICT - # and things requiring a fix in Tddium, omitted to avoid confusion: - - exclude: rpython/rlib/unicodedata/test/test_ucd.py # need wide build - - exclude: rpython/rlib/test/test_runicode.py # need wide build - - exclude: rpython/rlib/test/test_rsocket.py # not clear why fails - - exclude: pypy/module/cpyext/test/** # multiple failures due to truncated files; not clear why From noreply at buildbot.pypy.org Mon Apr 6 10:45:01 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 6 Apr 2015 10:45:01 +0200 (CEST) Subject: [pypy-commit] pypy more-rposix: Fix after merge Message-ID: <20150406084501.118A71C11B1@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: more-rposix Changeset: r76719:bb2928ec2881 Date: 2015-04-05 01:04 +0200 http://bitbucket.org/pypy/pypy/changeset/bb2928ec2881/ Log: Fix after merge diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -1284,10 +1284,7 @@ if not _WIN32: l_tmsbuf = lltype.malloc(TMSP.TO, flavor='raw') try: - result = c_times(l_tmsbuf) - result = rffi.cast(lltype.Signed, result) - if result == -1: - raise OSError(get_saved_errno(), "times failed") + handle_posix_error('times', c_times(l_tmsbuf)) return ( rffi.cast(lltype.Signed, l_tmsbuf.c_tms_utime) / CLOCK_TICKS_PER_SECOND, @@ -1323,7 +1320,7 @@ lltype.free(pexit, flavor='raw') lltype.free(pcreate, flavor='raw') -c_kill = external('kill', [rffi.PID_T, rffi.INT], rffi.INT +c_kill = external('kill', [rffi.PID_T, rffi.INT], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO) c_killpg = external('killpg', [rffi.INT, rffi.INT], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO) From noreply at buildbot.pypy.org Mon Apr 6 10:44:59 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 6 Apr 2015 10:44:59 +0200 (CEST) Subject: [pypy-commit] pypy more-rposix: hg merge default Message-ID: <20150406084459.C59F01C11B1@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: more-rposix Changeset: r76718:7c1b71e49c90 Date: 2015-04-04 18:58 +0200 http://bitbucket.org/pypy/pypy/changeset/7c1b71e49c90/ Log: hg merge default diff too long, truncating to 2000 out of 68117 lines diff --git a/.gitignore b/.gitignore --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,10 @@ bin/pypy-c include/*.h +include/numpy/ lib_pypy/ctypes_config_cache/_[^_]*_*.py +libpypy-c.* +pypy-c pypy/_cache pypy/doc/*.html pypy/doc/config/*.html @@ -18,4 +21,5 @@ pypy/translator/c/src/dtoa.o pypy/translator/goal/pypy-c pypy/translator/goal/target*-c -release/ \ No newline at end of file +release/ +rpython/_cache/ diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -3,14 +3,10 @@ d8ac7d23d3ec5f9a0fa1264972f74a010dbfd07f release-1.6 ff4af8f318821f7f5ca998613a60fca09aa137da release-1.7 07e08e9c885ca67d89bcc304e45a32346daea2fa release-2.0-beta-1 -9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm -9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm 20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0 -20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0 -0000000000000000000000000000000000000000 release-2.3.0 394146e9bb673514c61f0150ab2013ccf78e8de7 release-2.3 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1 -32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1 -0000000000000000000000000000000000000000 release-2.2=3.1 +10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0 +9c4588d731b7fe0b08669bd732c2b676cb0a8233 release-2.5.1 diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -28,7 +28,7 @@ DEALINGS IN THE SOFTWARE. -PyPy Copyright holders 2003-2014 +PyPy Copyright holders 2003-2015 ----------------------------------- Except when otherwise stated (look for LICENSE files or information at @@ -42,19 +42,19 @@ Amaury Forgeot d'Arc Samuele Pedroni Alex Gaynor + Brian Kearns + Matti Picus + Philip Jenvey Michael Hudson David Schneider - Matti Picus - Brian Kearns - Philip Jenvey Holger Krekel Christian Tismer Hakan Ardo Benjamin Peterson Manuel Jacob + Ronan Lamy Anders Chrigstrom Eric van Riet Paap - Ronan Lamy Wim Lavrijsen Richard Emslie Alexander Schremmer @@ -68,9 +68,9 @@ Camillo Bruni Laura Creighton Toon Verwaest + Romain Guillebert Leonardo Santagada Seo Sanghyeon - Romain Guillebert Justin Peel Ronny Pfannschmidt David Edelsohn @@ -91,15 +91,16 @@ Michal Bendowski Jan de Mooij stian + Tyler Wade Michael Foord Stephan Diehl - Tyler Wade Stefan Schwarzer Valentino Volonghi Tomek Meka Patrick Maupin Bob Ippolito Bruno Gola + David Malcolm Jean-Paul Calderone Timo Paulssen Squeaky @@ -108,18 +109,19 @@ Marius Gedminas Martin Matusiak Konstantin Lopuhin + Wenzhu Man John Witulski - Wenzhu Man + Laurence Tratt + Ivan Sichmann Freitas Greg Price Dario Bertini Mark Pearse Simon Cross - Ivan Sichmann Freitas Andreas Stührk + Stefano Rivera Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov - Stefano Rivera Paweł Piotr Przeradowski Paul deGrandis Ilya Osadchiy @@ -129,7 +131,6 @@ tav Taavi Burns Georg Brandl - Laurence Tratt Bert Freudenberg Stian Andreassen Wanja Saatkamp @@ -141,13 +142,12 @@ Jeremy Thurgood Rami Chowdhury Tobias Pape - David Malcolm Eugene Oden Henry Mason Vasily Kuznetsov Preston Timmons + David Ripton Jeff Terrace - David Ripton Dusty Phillips Lukas Renggli Guenter Jantzen @@ -166,13 +166,16 @@ Gintautas Miliauskas Michael Twomey Lucian Branescu Mihaila + Yichao Yu Gabriel Lavoie Olivier Dormond Jared Grubb Karl Bartel + Wouter van Heyst Brian Dorsey Victor Stinner Andrews Medina + anatoly techtonik Stuart Williams Jasper Schulz Christian Hudon @@ -182,12 +185,11 @@ Michael Cheng Justas Sadzevicius Gasper Zejn - anatoly techtonik Neil Shepperd + Stanislaw Halik Mikael Schönenberg Elmo M?ntynen Jonathan David Riehl - Stanislaw Halik Anders Qvist Corbin Simpson Chirag Jadwani @@ -196,10 +198,13 @@ Vincent Legoll Alan McIntyre Alexander Sedov + Attila Gobi Christopher Pope Christian Tismer Marc Abramowitz Dan Stromberg + Arjun Naik + Valentina Mukhamedzhanova Stefano Parmesan Alexis Daboville Jens-Uwe Mager @@ -213,8 +218,6 @@ Sylvain Thenault Nathan Taylor Vladimir Kryachko - Arjun Naik - Attila Gobi Jacek Generowicz Alejandro J. Cura Jacob Oscarson @@ -222,22 +225,23 @@ Ryan Gonzalez Ian Foote Kristjan Valur Jonsson + David Lievens Neil Blakey-Milner Lutz Paelike Lucio Torre Lars Wassermann - Valentina Mukhamedzhanova Henrik Vendelbo Dan Buch Miguel de Val Borro Artur Lisiecki Sergey Kishchenko - Yichao Yu Ignas Mikalajunas Christoph Gerum Martin Blais Lene Wagner Tomo Cocoa + Toni Mattis + Lucas Stadler roberto at goyle Yury V. Zaytsev Anna Katrina Dominguez @@ -265,23 +269,30 @@ Stephan Busemann Rafał Gałczyński Christian Muirhead + Berker Peksag James Lan shoma hosaka - Daniel Neuh?user - Matthew Miller + Daniel Neuhäuser + Ben Mather + halgari + Boglarka Vezer + Chris Pressey Buck Golemon Konrad Delong Dinu Gherman Chris Lambacher coolbutuseless at gmail.com + Jim Baker Rodrigo Araújo - Jim Baker + Nikolaos-Digenis Karagiannis James Robert Armin Ronacher Brett Cannon + Donald Stufft yrttyr aliceinwire OlivierBlanvillain + Dan Sanders Zooko Wilcox-O Hearn Tomer Chachamu Christopher Groskopf @@ -295,6 +306,7 @@ Markus Unterwaditzer Even Wiik Thomassen jbs + squeaky soareschen Kurt Griffiths Mike Bayer @@ -306,6 +318,7 @@ Anna Ravencroft Dan Crosta Julien Phalip + Roman Podoliaka Dan Loewenherz Heinrich-Heine University, Germany diff --git a/lib-python/2.7/CGIHTTPServer.py b/lib-python/2.7/CGIHTTPServer.py --- a/lib-python/2.7/CGIHTTPServer.py +++ b/lib-python/2.7/CGIHTTPServer.py @@ -106,16 +106,16 @@ def run_cgi(self): """Execute a CGI script.""" dir, rest = self.cgi_info - - i = rest.find('/') + path = dir + '/' + rest + i = path.find('/', len(dir)+1) while i >= 0: - nextdir = rest[:i] - nextrest = rest[i+1:] + nextdir = path[:i] + nextrest = path[i+1:] scriptdir = self.translate_path(nextdir) if os.path.isdir(scriptdir): dir, rest = nextdir, nextrest - i = rest.find('/') + i = path.find('/', len(dir)+1) else: break diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py --- a/lib-python/2.7/Cookie.py +++ b/lib-python/2.7/Cookie.py @@ -56,7 +56,7 @@ >>> C = Cookie.SmartCookie() [Note: Long-time users of Cookie.py will remember using -Cookie.Cookie() to create an Cookie object. Although deprecated, it +Cookie.Cookie() to create a Cookie object. Although deprecated, it is still supported by the code. See the Backward Compatibility notes for more information.] @@ -426,6 +426,8 @@ "version" : "Version", } + _flags = {'secure', 'httponly'} + def __init__(self): # Set defaults self.key = self.value = self.coded_value = None @@ -529,9 +531,11 @@ _LegalCharsPatt = r"[\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=]" _CookiePattern = re.compile( r"(?x)" # This is a Verbose pattern + r"\s*" # Optional whitespace at start of cookie r"(?P" # Start of group 'key' ""+ _LegalCharsPatt +"+?" # Any word of at least one letter, nongreedy r")" # End of group 'key' + r"(" # Optional group: there may not be a value. r"\s*=\s*" # Equal Sign r"(?P" # Start of group 'val' r'"(?:[^\\"]|\\.)*"' # Any doublequoted string @@ -540,7 +544,9 @@ r"|" # or ""+ _LegalCharsPatt +"*" # Any word or empty string r")" # End of group 'val' - r"\s*;?" # Probably ending in a semi-colon + r")?" # End of optional value group + r"\s*" # Any number of spaces. + r"(\s+|;|$)" # Ending either at space, semicolon, or EOS. ) @@ -585,8 +591,12 @@ def __setitem__(self, key, value): """Dictionary style assignment.""" - rval, cval = self.value_encode(value) - self.__set(key, rval, cval) + if isinstance(value, Morsel): + # allow assignment of constructed Morsels (e.g. for pickling) + dict.__setitem__(self, key, value) + else: + rval, cval = self.value_encode(value) + self.__set(key, rval, cval) # end __setitem__ def output(self, attrs=None, header="Set-Cookie:", sep="\015\012"): @@ -641,7 +651,7 @@ while 0 <= i < n: # Start looking for a cookie - match = patt.search(str, i) + match = patt.match(str, i) if not match: break # No more cookies K,V = match.group("key"), match.group("val") @@ -656,8 +666,12 @@ M[ K[1:] ] = V elif K.lower() in Morsel._reserved: if M: - M[ K ] = _unquote(V) - else: + if V is None: + if K.lower() in Morsel._flags: + M[K] = True + else: + M[K] = _unquote(V) + elif V is not None: rval, cval = self.value_decode(V) self.__set(K, rval, cval) M = self[K] diff --git a/lib-python/2.7/SocketServer.py b/lib-python/2.7/SocketServer.py --- a/lib-python/2.7/SocketServer.py +++ b/lib-python/2.7/SocketServer.py @@ -416,8 +416,12 @@ self.socket = socket.socket(self.address_family, self.socket_type) if bind_and_activate: - self.server_bind() - self.server_activate() + try: + self.server_bind() + self.server_activate() + except: + self.server_close() + raise def server_bind(self): """Called by constructor to bind the socket. diff --git a/lib-python/2.7/_abcoll.py b/lib-python/2.7/_abcoll.py --- a/lib-python/2.7/_abcoll.py +++ b/lib-python/2.7/_abcoll.py @@ -143,7 +143,7 @@ methods except for __contains__, __iter__ and __len__. To override the comparisons (presumably for speed, as the - semantics are fixed), all you have to do is redefine __le__ and + semantics are fixed), redefine __le__ and __ge__, then the other operations will automatically follow suit. """ diff --git a/lib-python/2.7/argparse.py b/lib-python/2.7/argparse.py --- a/lib-python/2.7/argparse.py +++ b/lib-python/2.7/argparse.py @@ -1089,7 +1089,14 @@ # parse all the remaining options into the namespace # store any unrecognized options on the object, so that the top # level parser can decide what to do with them - namespace, arg_strings = parser.parse_known_args(arg_strings, namespace) + + # In case this subparser defines new defaults, we parse them + # in a new namespace object and then update the original + # namespace for the relevant parts. + subnamespace, arg_strings = parser.parse_known_args(arg_strings, None) + for key, value in vars(subnamespace).items(): + setattr(namespace, key, value) + if arg_strings: vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, []) getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings) diff --git a/lib-python/2.7/asynchat.py b/lib-python/2.7/asynchat.py --- a/lib-python/2.7/asynchat.py +++ b/lib-python/2.7/asynchat.py @@ -46,12 +46,17 @@ you - by calling your self.found_terminator() method. """ +import asyncore +import errno import socket -import asyncore from collections import deque from sys import py3kwarning from warnings import filterwarnings, catch_warnings +_BLOCKING_IO_ERRORS = (errno.EAGAIN, errno.EALREADY, errno.EINPROGRESS, + errno.EWOULDBLOCK) + + class async_chat (asyncore.dispatcher): """This is an abstract class. You must derive from this class, and add the two methods collect_incoming_data() and found_terminator()""" @@ -109,6 +114,8 @@ try: data = self.recv (self.ac_in_buffer_size) except socket.error, why: + if why.args[0] in _BLOCKING_IO_ERRORS: + return self.handle_error() return diff --git a/lib-python/2.7/bsddb/test/test_queue.py b/lib-python/2.7/bsddb/test/test_queue.py --- a/lib-python/2.7/bsddb/test/test_queue.py +++ b/lib-python/2.7/bsddb/test/test_queue.py @@ -10,6 +10,7 @@ #---------------------------------------------------------------------- + at unittest.skip("fails on Windows; see issue 22943") class SimpleQueueTestCase(unittest.TestCase): def setUp(self): self.filename = get_new_database_path() diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py --- a/lib-python/2.7/collections.py +++ b/lib-python/2.7/collections.py @@ -17,6 +17,10 @@ except ImportError: assert '__pypy__' not in _sys.builtin_module_names newdict = lambda _ : {} +try: + from __pypy__ import reversed_dict +except ImportError: + reversed_dict = lambda d: reversed(d.keys()) try: from thread import get_ident as _get_ident @@ -29,142 +33,35 @@ ################################################################################ class OrderedDict(dict): - 'Dictionary that remembers insertion order' - # An inherited dict maps keys to values. - # The inherited dict provides __getitem__, __len__, __contains__, and get. - # The remaining methods are order-aware. - # Big-O running times for all methods are the same as regular dictionaries. + '''Dictionary that remembers insertion order. - # The internal self.__map dict maps keys to links in a doubly linked list. - # The circular doubly linked list starts and ends with a sentinel element. - # The sentinel element never gets deleted (this simplifies the algorithm). - # Each link is stored as a list of length three: [PREV, NEXT, KEY]. + In PyPy all dicts are ordered anyway. This is mostly useful as a + placeholder to mean "this dict must be ordered even on CPython". - def __init__(self, *args, **kwds): - '''Initialize an ordered dictionary. The signature is the same as - regular dictionaries, but keyword arguments are not recommended because - their insertion order is arbitrary. - - ''' - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - try: - self.__root - except AttributeError: - self.__root = root = [] # sentinel node - root[:] = [root, root, None] - self.__map = {} - self.__update(*args, **kwds) - - def __setitem__(self, key, value, dict_setitem=dict.__setitem__): - 'od.__setitem__(i, y) <==> od[i]=y' - # Setting a new item creates a new link at the end of the linked list, - # and the inherited dictionary is updated with the new key/value pair. - if key not in self: - root = self.__root - last = root[0] - last[1] = root[0] = self.__map[key] = [last, root, key] - return dict_setitem(self, key, value) - - def __delitem__(self, key, dict_delitem=dict.__delitem__): - 'od.__delitem__(y) <==> del od[y]' - # Deleting an existing item uses self.__map to find the link which gets - # removed by updating the links in the predecessor and successor nodes. - dict_delitem(self, key) - link_prev, link_next, _ = self.__map.pop(key) - link_prev[1] = link_next # update link_prev[NEXT] - link_next[0] = link_prev # update link_next[PREV] - - def __iter__(self): - 'od.__iter__() <==> iter(od)' - # Traverse the linked list in order. - root = self.__root - curr = root[1] # start at the first node - while curr is not root: - yield curr[2] # yield the curr[KEY] - curr = curr[1] # move to next node + Known difference: iterating over an OrderedDict which is being + concurrently modified raises RuntimeError in PyPy. In CPython + instead we get some behavior that appears reasonable in some + cases but is nonsensical in other cases. This is officially + forbidden by the CPython docs, so we forbid it explicitly for now. + ''' def __reversed__(self): - 'od.__reversed__() <==> reversed(od)' - # Traverse the linked list in reverse order. - root = self.__root - curr = root[0] # start at the last node - while curr is not root: - yield curr[2] # yield the curr[KEY] - curr = curr[0] # move to previous node - - def clear(self): - 'od.clear() -> None. Remove all items from od.' - root = self.__root - root[:] = [root, root, None] - self.__map.clear() - dict.clear(self) - - # -- the following methods do not depend on the internal structure -- - - def keys(self): - 'od.keys() -> list of keys in od' - return list(self) - - def values(self): - 'od.values() -> list of values in od' - return [self[key] for key in self] - - def items(self): - 'od.items() -> list of (key, value) pairs in od' - return [(key, self[key]) for key in self] - - def iterkeys(self): - 'od.iterkeys() -> an iterator over the keys in od' - return iter(self) - - def itervalues(self): - 'od.itervalues -> an iterator over the values in od' - for k in self: - yield self[k] - - def iteritems(self): - 'od.iteritems -> an iterator over the (key, value) pairs in od' - for k in self: - yield (k, self[k]) - - update = MutableMapping.update - - __update = update # let subclasses override update without breaking __init__ - - __marker = object() - - def pop(self, key, default=__marker): - '''od.pop(k[,d]) -> v, remove specified key and return the corresponding - value. If key is not found, d is returned if given, otherwise KeyError - is raised. - - ''' - if key in self: - result = self[key] - del self[key] - return result - if default is self.__marker: - raise KeyError(key) - return default - - def setdefault(self, key, default=None): - 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' - if key in self: - return self[key] - self[key] = default - return default + return reversed_dict(self) def popitem(self, last=True): '''od.popitem() -> (k, v), return and remove a (key, value) pair. Pairs are returned in LIFO order if last is true or FIFO order if false. ''' - if not self: - raise KeyError('dictionary is empty') - key = next(reversed(self) if last else iter(self)) - value = self.pop(key) - return key, value + if last: + return dict.popitem(self) + else: + it = dict.__iter__(self) + try: + k = it.next() + except StopIteration: + raise KeyError('dictionary is empty') + return (k, self.pop(k)) def __repr__(self, _repr_running={}): 'od.__repr__() <==> repr(od)' @@ -183,8 +80,6 @@ 'Return state information for pickling' items = [[k, self[k]] for k in self] inst_dict = vars(self).copy() - for k in vars(OrderedDict()): - inst_dict.pop(k, None) if inst_dict: return (self.__class__, (items,), inst_dict) return self.__class__, (items,) @@ -193,17 +88,6 @@ 'od.copy() -> a shallow copy of od' return self.__class__(self) - @classmethod - def fromkeys(cls, iterable, value=None): - '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S. - If not specified, the value defaults to None. - - ''' - self = cls() - for key in iterable: - self[key] = value - return self - def __eq__(self, other): '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive while comparison to a regular mapping is order-insensitive. diff --git a/lib-python/2.7/cookielib.py b/lib-python/2.7/cookielib.py --- a/lib-python/2.7/cookielib.py +++ b/lib-python/2.7/cookielib.py @@ -1719,12 +1719,12 @@ def __repr__(self): r = [] for cookie in self: r.append(repr(cookie)) - return "<%s[%s]>" % (self.__class__, ", ".join(r)) + return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r)) def __str__(self): r = [] for cookie in self: r.append(str(cookie)) - return "<%s[%s]>" % (self.__class__, ", ".join(r)) + return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r)) # derives from IOError for backwards-compatibility with Python 2.4.0 diff --git a/lib-python/2.7/ctypes/test/test_frombuffer.py b/lib-python/2.7/ctypes/test/test_frombuffer.py --- a/lib-python/2.7/ctypes/test/test_frombuffer.py +++ b/lib-python/2.7/ctypes/test/test_frombuffer.py @@ -2,7 +2,6 @@ import array import gc import unittest -from ctypes.test import xfail class X(Structure): _fields_ = [("c_int", c_int)] @@ -11,7 +10,6 @@ self._init_called = True class Test(unittest.TestCase): - @xfail def test_fom_buffer(self): a = array.array("i", range(16)) x = (c_int * 16).from_buffer(a) @@ -34,10 +32,9 @@ del a; gc.collect(); gc.collect(); gc.collect() self.assertEqual(x[:], expected) - self.assertRaises(TypeError, + self.assertRaises((TypeError, ValueError), (c_char * 16).from_buffer, "a" * 16) - @xfail def test_fom_buffer_with_offset(self): a = array.array("i", range(16)) x = (c_int * 15).from_buffer(a, sizeof(c_int)) @@ -46,7 +43,6 @@ self.assertRaises(ValueError, lambda: (c_int * 16).from_buffer(a, sizeof(c_int))) self.assertRaises(ValueError, lambda: (c_int * 1).from_buffer(a, 16 * sizeof(c_int))) - @xfail def test_from_buffer_copy(self): a = array.array("i", range(16)) x = (c_int * 16).from_buffer_copy(a) @@ -71,7 +67,6 @@ x = (c_char * 16).from_buffer_copy("a" * 16) self.assertEqual(x[:], "a" * 16) - @xfail def test_fom_buffer_copy_with_offset(self): a = array.array("i", range(16)) x = (c_int * 15).from_buffer_copy(a, sizeof(c_int)) diff --git a/lib-python/2.7/ctypes/test/test_pointers.py b/lib-python/2.7/ctypes/test/test_pointers.py --- a/lib-python/2.7/ctypes/test/test_pointers.py +++ b/lib-python/2.7/ctypes/test/test_pointers.py @@ -7,6 +7,8 @@ c_long, c_ulong, c_longlong, c_ulonglong, c_double, c_float] python_types = [int, int, int, int, int, long, int, long, long, long, float, float] +LargeNamedType = type('T' * 2 ** 25, (Structure,), {}) +large_string = 'T' * 2 ** 25 class PointersTestCase(unittest.TestCase): @@ -188,5 +190,11 @@ mth = WINFUNCTYPE(None)(42, "name", (), None) self.assertEqual(bool(mth), True) + def test_pointer_type_name(self): + self.assertTrue(POINTER(LargeNamedType)) + + def test_pointer_type_str_name(self): + self.assertTrue(POINTER(large_string)) + if __name__ == '__main__': unittest.main() diff --git a/lib-python/2.7/ctypes/test/test_python_api.py b/lib-python/2.7/ctypes/test/test_python_api.py --- a/lib-python/2.7/ctypes/test/test_python_api.py +++ b/lib-python/2.7/ctypes/test/test_python_api.py @@ -46,8 +46,8 @@ # This test is unreliable, because it is possible that code in # unittest changes the refcount of the '42' integer. So, it # is disabled by default. - @requires("refcount") def test_PyInt_Long(self): + requires("refcount") ref42 = grc(42) pythonapi.PyInt_FromLong.restype = py_object self.assertEqual(pythonapi.PyInt_FromLong(42), 42) diff --git a/lib-python/2.7/ctypes/test/test_win32.py b/lib-python/2.7/ctypes/test/test_win32.py --- a/lib-python/2.7/ctypes/test/test_win32.py +++ b/lib-python/2.7/ctypes/test/test_win32.py @@ -38,8 +38,11 @@ @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') class FunctionCallTestCase(unittest.TestCase): - @requires("SEH") + @unittest.skipUnless('MSC' in sys.version, "SEH only supported by MSC") + @unittest.skipIf(sys.executable.endswith('_d.exe'), + "SEH not enabled in debug builds") def test_SEH(self): + requires("SEH") # Call functions with invalid arguments, and make sure # that access violations are trapped and raise an # exception. @@ -87,9 +90,29 @@ dll = CDLL(_ctypes_test.__file__) - pt = POINT(10, 10) - rect = RECT(0, 0, 20, 20) - self.assertEqual(1, dll.PointInRect(byref(rect), pt)) + pt = POINT(15, 25) + left = c_long.in_dll(dll, 'left') + top = c_long.in_dll(dll, 'top') + right = c_long.in_dll(dll, 'right') + bottom = c_long.in_dll(dll, 'bottom') + rect = RECT(left, top, right, bottom) + PointInRect = dll.PointInRect + PointInRect.argtypes = [POINTER(RECT), POINT] + self.assertEqual(1, PointInRect(byref(rect), pt)) + + ReturnRect = dll.ReturnRect + ReturnRect.argtypes = [c_int, RECT, POINTER(RECT), POINT, RECT, + POINTER(RECT), POINT, RECT] + ReturnRect.restype = RECT + for i in range(4): + ret = ReturnRect(i, rect, pointer(rect), pt, rect, + byref(rect), pt, rect) + # the c function will check and modify ret if something is + # passed in improperly + self.assertEqual(ret.left, left.value) + self.assertEqual(ret.right, right.value) + self.assertEqual(ret.top, top.value) + self.assertEqual(ret.bottom, bottom.value) if __name__ == '__main__': unittest.main() diff --git a/lib-python/2.7/decimal.py b/lib-python/2.7/decimal.py --- a/lib-python/2.7/decimal.py +++ b/lib-python/2.7/decimal.py @@ -136,7 +136,6 @@ __version__ = '1.70' # Highest version of the spec this complies with -import copy as _copy import math as _math import numbers as _numbers @@ -3665,6 +3664,8 @@ if self._is_special: sign = _format_sign(self._sign, spec) body = str(self.copy_abs()) + if spec['type'] == '%': + body += '%' return _format_align(sign, body, spec) # a type of None defaults to 'g' or 'G', depending on context @@ -6033,7 +6034,10 @@ format_dict['decimal_point'] = '.' # record whether return type should be str or unicode - format_dict['unicode'] = isinstance(format_spec, unicode) + try: + format_dict['unicode'] = isinstance(format_spec, unicode) + except NameError: + format_dict['unicode'] = False return format_dict diff --git a/lib-python/2.7/distutils/__init__.py b/lib-python/2.7/distutils/__init__.py --- a/lib-python/2.7/distutils/__init__.py +++ b/lib-python/2.7/distutils/__init__.py @@ -15,5 +15,5 @@ # Updated automatically by the Python release process. # #--start constants-- -__version__ = "2.7.8" +__version__ = "2.7.9" #--end constants-- diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py --- a/lib-python/2.7/distutils/command/build_ext.py +++ b/lib-python/2.7/distutils/command/build_ext.py @@ -245,7 +245,7 @@ # Python's library directory must be appended to library_dirs # See Issues: #1600860, #4366 if (sysconfig.get_config_var('Py_ENABLE_SHARED')): - if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")): + if not sysconfig.python_build: # building third party extensions self.library_dirs.append(sysconfig.get_config_var('LIBDIR')) else: diff --git a/lib-python/2.7/distutils/command/upload.py b/lib-python/2.7/distutils/command/upload.py --- a/lib-python/2.7/distutils/command/upload.py +++ b/lib-python/2.7/distutils/command/upload.py @@ -136,8 +136,8 @@ # Build up the MIME payload for the POST data boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254' - sep_boundary = '\n--' + boundary - end_boundary = sep_boundary + '--' + sep_boundary = '\r\n--' + boundary + end_boundary = sep_boundary + '--\r\n' body = StringIO.StringIO() for key, value in data.items(): # handle multiple entries for the same name @@ -151,14 +151,13 @@ fn = "" body.write(sep_boundary) - body.write('\nContent-Disposition: form-data; name="%s"'%key) + body.write('\r\nContent-Disposition: form-data; name="%s"' % key) body.write(fn) - body.write("\n\n") + body.write("\r\n\r\n") body.write(value) if value and value[-1] == '\r': body.write('\n') # write an extra newline (lurve Macs) body.write(end_boundary) - body.write("\n") body = body.getvalue() self.announce("Submitting %s to %s" % (filename, self.repository), log.INFO) diff --git a/lib-python/2.7/distutils/file_util.py b/lib-python/2.7/distutils/file_util.py --- a/lib-python/2.7/distutils/file_util.py +++ b/lib-python/2.7/distutils/file_util.py @@ -85,7 +85,8 @@ (os.symlink) instead of copying: set it to "hard" or "sym"; if it is None (the default), files are copied. Don't set 'link' on systems that don't support it: 'copy_file()' doesn't check if hard or symbolic - linking is available. + linking is available. If hardlink fails, falls back to + _copy_file_contents(). Under Mac OS, uses the native file copy function in macostools; on other systems, uses '_copy_file_contents()' to copy file contents. @@ -137,24 +138,31 @@ # (Unix only, of course, but that's the caller's responsibility) if link == 'hard': if not (os.path.exists(dst) and os.path.samefile(src, dst)): - os.link(src, dst) + try: + os.link(src, dst) + return (dst, 1) + except OSError: + # If hard linking fails, fall back on copying file + # (some special filesystems don't support hard linking + # even under Unix, see issue #8876). + pass elif link == 'sym': if not (os.path.exists(dst) and os.path.samefile(src, dst)): os.symlink(src, dst) + return (dst, 1) # Otherwise (non-Mac, not linking), copy the file contents and # (optionally) copy the times and mode. - else: - _copy_file_contents(src, dst) - if preserve_mode or preserve_times: - st = os.stat(src) + _copy_file_contents(src, dst) + if preserve_mode or preserve_times: + st = os.stat(src) - # According to David Ascher , utime() should be done - # before chmod() (at least under NT). - if preserve_times: - os.utime(dst, (st[ST_ATIME], st[ST_MTIME])) - if preserve_mode: - os.chmod(dst, S_IMODE(st[ST_MODE])) + # According to David Ascher , utime() should be done + # before chmod() (at least under NT). + if preserve_times: + os.utime(dst, (st[ST_ATIME], st[ST_MTIME])) + if preserve_mode: + os.chmod(dst, S_IMODE(st[ST_MODE])) return (dst, 1) diff --git a/lib-python/2.7/distutils/sysconfig_cpython.py b/lib-python/2.7/distutils/sysconfig_cpython.py --- a/lib-python/2.7/distutils/sysconfig_cpython.py +++ b/lib-python/2.7/distutils/sysconfig_cpython.py @@ -165,7 +165,8 @@ # version and build tools may not support the same set # of CPU architectures for universal builds. global _config_vars - if not _config_vars.get('CUSTOMIZED_OSX_COMPILER', ''): + # Use get_config_var() to ensure _config_vars is initialized. + if not get_config_var('CUSTOMIZED_OSX_COMPILER'): import _osx_support _osx_support.customize_compiler(_config_vars) _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True' diff --git a/lib-python/2.7/distutils/tests/test_bdist_rpm.py b/lib-python/2.7/distutils/tests/test_bdist_rpm.py --- a/lib-python/2.7/distutils/tests/test_bdist_rpm.py +++ b/lib-python/2.7/distutils/tests/test_bdist_rpm.py @@ -25,6 +25,7 @@ """ class BuildRpmTestCase(support.TempdirManager, + support.EnvironGuard, support.LoggingSilencer, unittest.TestCase): @@ -50,6 +51,7 @@ def test_quiet(self): # let's create a package tmp_dir = self.mkdtemp() + os.environ['HOME'] = tmp_dir # to confine dir '.rpmdb' creation pkg_dir = os.path.join(tmp_dir, 'foo') os.mkdir(pkg_dir) self.write_file((pkg_dir, 'setup.py'), SETUP_PY) @@ -92,6 +94,7 @@ def test_no_optimize_flag(self): # let's create a package that brakes bdist_rpm tmp_dir = self.mkdtemp() + os.environ['HOME'] = tmp_dir # to confine dir '.rpmdb' creation pkg_dir = os.path.join(tmp_dir, 'foo') os.mkdir(pkg_dir) self.write_file((pkg_dir, 'setup.py'), SETUP_PY) diff --git a/lib-python/2.7/distutils/tests/test_dist.py b/lib-python/2.7/distutils/tests/test_dist.py --- a/lib-python/2.7/distutils/tests/test_dist.py +++ b/lib-python/2.7/distutils/tests/test_dist.py @@ -11,7 +11,7 @@ from distutils.dist import Distribution, fix_help_options from distutils.cmd import Command import distutils.dist -from test.test_support import TESTFN, captured_stdout, run_unittest +from test.test_support import TESTFN, captured_stdout, run_unittest, unlink from distutils.tests import support @@ -64,6 +64,7 @@ with open(TESTFN, "w") as f: f.write("[global]\n") f.write("command_packages = foo.bar, splat") + self.addCleanup(unlink, TESTFN) files = [TESTFN] sys.argv.append("build") diff --git a/lib-python/2.7/distutils/tests/test_file_util.py b/lib-python/2.7/distutils/tests/test_file_util.py --- a/lib-python/2.7/distutils/tests/test_file_util.py +++ b/lib-python/2.7/distutils/tests/test_file_util.py @@ -8,6 +8,11 @@ from distutils.tests import support from test.test_support import run_unittest + +requires_os_link = unittest.skipUnless(hasattr(os, "link"), + "test requires os.link()") + + class FileUtilTestCase(support.TempdirManager, unittest.TestCase): def _log(self, msg, *args): @@ -74,6 +79,44 @@ copy_file(foo, dst_dir) self.assertTrue(os.path.exists(os.path.join(dst_dir, 'foo'))) + @requires_os_link + def test_copy_file_hard_link(self): + with open(self.source, 'w') as f: + f.write('some content') + st = os.stat(self.source) + copy_file(self.source, self.target, link='hard') + st2 = os.stat(self.source) + st3 = os.stat(self.target) + self.assertTrue(os.path.samestat(st, st2), (st, st2)) + self.assertTrue(os.path.samestat(st2, st3), (st2, st3)) + with open(self.source, 'r') as f: + self.assertEqual(f.read(), 'some content') + + @requires_os_link + def test_copy_file_hard_link_failure(self): + # If hard linking fails, copy_file() falls back on copying file + # (some special filesystems don't support hard linking even under + # Unix, see issue #8876). + with open(self.source, 'w') as f: + f.write('some content') + st = os.stat(self.source) + def _os_link(*args): + raise OSError(0, "linking unsupported") + old_link = os.link + os.link = _os_link + try: + copy_file(self.source, self.target, link='hard') + finally: + os.link = old_link + st2 = os.stat(self.source) + st3 = os.stat(self.target) + self.assertTrue(os.path.samestat(st, st2), (st, st2)) + self.assertFalse(os.path.samestat(st2, st3), (st2, st3)) + for fn in (self.source, self.target): + with open(fn, 'r') as f: + self.assertEqual(f.read(), 'some content') + + def test_suite(): return unittest.makeSuite(FileUtilTestCase) diff --git a/lib-python/2.7/distutils/tests/test_sysconfig.py b/lib-python/2.7/distutils/tests/test_sysconfig.py --- a/lib-python/2.7/distutils/tests/test_sysconfig.py +++ b/lib-python/2.7/distutils/tests/test_sysconfig.py @@ -3,6 +3,9 @@ import test import unittest import shutil +import subprocess +import sys +import textwrap from distutils import sysconfig from distutils.tests import support @@ -99,6 +102,24 @@ self.assertEqual(global_sysconfig.get_config_var('LDSHARED'), sysconfig.get_config_var('LDSHARED')) self.assertEqual(global_sysconfig.get_config_var('CC'), sysconfig.get_config_var('CC')) + def test_customize_compiler_before_get_config_vars(self): + # Issue #21923: test that a Distribution compiler + # instance can be called without an explicit call to + # get_config_vars(). + with open(TESTFN, 'w') as f: + f.writelines(textwrap.dedent('''\ + from distutils.core import Distribution + config = Distribution().get_command_obj('config') + # try_compile may pass or it may fail if no compiler + # is found but it should not raise an exception. + rc = config.try_compile('int x;') + ''')) + p = subprocess.Popen([str(sys.executable), TESTFN], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True) + outs, errs = p.communicate() + self.assertEqual(0, p.returncode, "Subprocess failed: " + outs) def test_suite(): diff --git a/lib-python/2.7/distutils/tests/test_upload.py b/lib-python/2.7/distutils/tests/test_upload.py --- a/lib-python/2.7/distutils/tests/test_upload.py +++ b/lib-python/2.7/distutils/tests/test_upload.py @@ -119,7 +119,7 @@ # what did we send ? self.assertIn('dédé', self.last_open.req.data) headers = dict(self.last_open.req.headers) - self.assertEqual(headers['Content-length'], '2085') + self.assertEqual(headers['Content-length'], '2159') self.assertTrue(headers['Content-type'].startswith('multipart/form-data')) self.assertEqual(self.last_open.req.get_method(), 'POST') self.assertEqual(self.last_open.req.get_full_url(), diff --git a/lib-python/2.7/distutils/unixccompiler.py b/lib-python/2.7/distutils/unixccompiler.py --- a/lib-python/2.7/distutils/unixccompiler.py +++ b/lib-python/2.7/distutils/unixccompiler.py @@ -58,7 +58,7 @@ executables = {'preprocessor' : None, 'compiler' : ["cc"], 'compiler_so' : ["cc"], - 'compiler_cxx' : ["cc"], + 'compiler_cxx' : ["c++"], # pypy: changed, 'cc' is bogus 'linker_so' : ["cc", "-shared"], 'linker_exe' : ["cc"], 'archiver' : ["ar", "-cr"], diff --git a/lib-python/2.7/doctest.py b/lib-python/2.7/doctest.py --- a/lib-python/2.7/doctest.py +++ b/lib-python/2.7/doctest.py @@ -216,7 +216,7 @@ # get_data() opens files as 'rb', so one must do the equivalent # conversion as universal newlines would do. return file_contents.replace(os.linesep, '\n'), filename - with open(filename) as f: + with open(filename, 'U') as f: return f.read(), filename # Use sys.stdout encoding for ouput. diff --git a/lib-python/2.7/email/feedparser.py b/lib-python/2.7/email/feedparser.py --- a/lib-python/2.7/email/feedparser.py +++ b/lib-python/2.7/email/feedparser.py @@ -49,8 +49,8 @@ simple abstraction -- it parses until EOF closes the current message. """ def __init__(self): - # The last partial line pushed into this object. - self._partial = '' + # Chunks of the last partial line pushed into this object. + self._partial = [] # The list of full, pushed lines, in reverse order self._lines = [] # The stack of false-EOF checking predicates. @@ -66,8 +66,8 @@ def close(self): # Don't forget any trailing partial line. - self._lines.append(self._partial) - self._partial = '' + self.pushlines(''.join(self._partial).splitlines(True)) + self._partial = [] self._closed = True def readline(self): @@ -95,8 +95,29 @@ def push(self, data): """Push some new data into this object.""" - # Handle any previous leftovers - data, self._partial = self._partial + data, '' + # Crack into lines, but preserve the linesep characters on the end of each + parts = data.splitlines(True) + + if not parts or not parts[0].endswith(('\n', '\r')): + # No new complete lines, so just accumulate partials + self._partial += parts + return + + if self._partial: + # If there are previous leftovers, complete them now + self._partial.append(parts[0]) + parts[0:1] = ''.join(self._partial).splitlines(True) + del self._partial[:] + + # If the last element of the list does not end in a newline, then treat + # it as a partial line. We only check for '\n' here because a line + # ending with '\r' might be a line that was split in the middle of a + # '\r\n' sequence (see bugs 1555570 and 1721862). + if not parts[-1].endswith('\n'): + self._partial = [parts.pop()] + self.pushlines(parts) + + def pushlines(self, lines): # Crack into lines, but preserve the newlines on the end of each parts = NLCRE_crack.split(data) # The *ahem* interesting behaviour of re.split when supplied grouping diff --git a/lib-python/2.7/email/mime/nonmultipart.py b/lib-python/2.7/email/mime/nonmultipart.py --- a/lib-python/2.7/email/mime/nonmultipart.py +++ b/lib-python/2.7/email/mime/nonmultipart.py @@ -12,7 +12,7 @@ class MIMENonMultipart(MIMEBase): - """Base class for MIME multipart/* type messages.""" + """Base class for MIME non-multipart type messages.""" def attach(self, payload): # The public API prohibits attaching multiple subparts to MIMEBase diff --git a/lib-python/2.7/email/test/test_email.py b/lib-python/2.7/email/test/test_email.py --- a/lib-python/2.7/email/test/test_email.py +++ b/lib-python/2.7/email/test/test_email.py @@ -11,6 +11,7 @@ import warnings import textwrap from cStringIO import StringIO +from random import choice import email @@ -2578,16 +2579,64 @@ bsf.push(il) nt += n n1 = 0 - while True: - ol = bsf.readline() - if ol == NeedMoreData: - break + for ol in iter(bsf.readline, NeedMoreData): om.append(ol) n1 += 1 self.assertEqual(n, n1) self.assertEqual(len(om), nt) self.assertEqual(''.join([il for il, n in imt]), ''.join(om)) + def test_push_random(self): + from email.feedparser import BufferedSubFile, NeedMoreData + + n = 10000 + chunksize = 5 + chars = 'abcd \t\r\n' + + s = ''.join(choice(chars) for i in range(n)) + '\n' + target = s.splitlines(True) + + bsf = BufferedSubFile() + lines = [] + for i in range(0, len(s), chunksize): + chunk = s[i:i+chunksize] + bsf.push(chunk) + lines.extend(iter(bsf.readline, NeedMoreData)) + self.assertEqual(lines, target) + + +class TestFeedParsers(TestEmailBase): + + def parse(self, chunks): + from email.feedparser import FeedParser + feedparser = FeedParser() + for chunk in chunks: + feedparser.feed(chunk) + return feedparser.close() + + def test_newlines(self): + m = self.parse(['a:\nb:\rc:\r\nd:\n']) + self.assertEqual(m.keys(), ['a', 'b', 'c', 'd']) + m = self.parse(['a:\nb:\rc:\r\nd:']) + self.assertEqual(m.keys(), ['a', 'b', 'c', 'd']) + m = self.parse(['a:\rb', 'c:\n']) + self.assertEqual(m.keys(), ['a', 'bc']) + m = self.parse(['a:\r', 'b:\n']) + self.assertEqual(m.keys(), ['a', 'b']) + m = self.parse(['a:\r', '\nb:\n']) + self.assertEqual(m.keys(), ['a', 'b']) + + def test_long_lines(self): + # Expected peak memory use on 32-bit platform: 4*N*M bytes. + M, N = 1000, 20000 + m = self.parse(['a:b\n\n'] + ['x'*M] * N) + self.assertEqual(m.items(), [('a', 'b')]) + self.assertEqual(m.get_payload(), 'x'*M*N) + m = self.parse(['a:b\r\r'] + ['x'*M] * N) + self.assertEqual(m.items(), [('a', 'b')]) + self.assertEqual(m.get_payload(), 'x'*M*N) + m = self.parse(['a:\r', 'b: '] + ['x'*M] * N) + self.assertEqual(m.items(), [('a', ''), ('b', 'x'*M*N)]) class TestParsers(TestEmailBase): @@ -3180,7 +3229,6 @@ self.assertEqual(res, '=?iso-8859-2?q?abc?=') self.assertIsInstance(res, str) - # Test RFC 2231 header parameters (en/de)coding class TestRFC2231(TestEmailBase): def test_get_param(self): diff --git a/lib-python/2.7/ensurepip/__init__.py b/lib-python/2.7/ensurepip/__init__.py new file mode 100644 --- /dev/null +++ b/lib-python/2.7/ensurepip/__init__.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python2 +from __future__ import print_function + +import os +import os.path +import pkgutil +import shutil +import sys +import tempfile + + +__all__ = ["version", "bootstrap"] + + +_SETUPTOOLS_VERSION = "7.0" + +_PIP_VERSION = "1.5.6" + +# pip currently requires ssl support, so we try to provide a nicer +# error message when that is missing (http://bugs.python.org/issue19744) +_MISSING_SSL_MESSAGE = ("pip {} requires SSL/TLS".format(_PIP_VERSION)) +try: + import ssl +except ImportError: + ssl = None + + def _require_ssl_for_pip(): + raise RuntimeError(_MISSING_SSL_MESSAGE) +else: + def _require_ssl_for_pip(): + pass + +_PROJECTS = [ + ("setuptools", _SETUPTOOLS_VERSION), + ("pip", _PIP_VERSION), +] + + +def _run_pip(args, additional_paths=None): + # Add our bundled software to the sys.path so we can import it + if additional_paths is not None: + sys.path = additional_paths + sys.path + + # Install the bundled software + import pip + pip.main(args) + + +def version(): + """ + Returns a string specifying the bundled version of pip. + """ + return _PIP_VERSION + + +def _disable_pip_configuration_settings(): + # We deliberately ignore all pip environment variables + # when invoking pip + # See http://bugs.python.org/issue19734 for details + keys_to_remove = [k for k in os.environ if k.startswith("PIP_")] + for k in keys_to_remove: + del os.environ[k] + # We also ignore the settings in the default pip configuration file + # See http://bugs.python.org/issue20053 for details + os.environ['PIP_CONFIG_FILE'] = os.devnull + + +def bootstrap(root=None, upgrade=False, user=False, + altinstall=False, default_pip=True, + verbosity=0): + """ + Bootstrap pip into the current Python installation (or the given root + directory). + + Note that calling this function will alter both sys.path and os.environ. + """ + if altinstall and default_pip: + raise ValueError("Cannot use altinstall and default_pip together") + + _require_ssl_for_pip() + _disable_pip_configuration_settings() + + # By default, installing pip and setuptools installs all of the + # following scripts (X.Y == running Python version): + # + # pip, pipX, pipX.Y, easy_install, easy_install-X.Y + # + # pip 1.5+ allows ensurepip to request that some of those be left out + if altinstall: + # omit pip, pipX and easy_install + os.environ["ENSUREPIP_OPTIONS"] = "altinstall" + elif not default_pip: + # omit pip and easy_install + os.environ["ENSUREPIP_OPTIONS"] = "install" + + tmpdir = tempfile.mkdtemp() + try: + # Put our bundled wheels into a temporary directory and construct the + # additional paths that need added to sys.path + additional_paths = [] + for project, version in _PROJECTS: + wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version) + whl = pkgutil.get_data( + "ensurepip", + "_bundled/{}".format(wheel_name), + ) + with open(os.path.join(tmpdir, wheel_name), "wb") as fp: + fp.write(whl) + + additional_paths.append(os.path.join(tmpdir, wheel_name)) + + # Construct the arguments to be passed to the pip command + args = ["install", "--no-index", "--find-links", tmpdir] + if root: + args += ["--root", root] + if upgrade: + args += ["--upgrade"] + if user: + args += ["--user"] + if verbosity: + args += ["-" + "v" * verbosity] + + _run_pip(args + [p[0] for p in _PROJECTS], additional_paths) + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + +def _uninstall_helper(verbosity=0): + """Helper to support a clean default uninstall process on Windows + + Note that calling this function may alter os.environ. + """ + # Nothing to do if pip was never installed, or has been removed + try: + import pip + except ImportError: + return + + # If the pip version doesn't match the bundled one, leave it alone + if pip.__version__ != _PIP_VERSION: + msg = ("ensurepip will only uninstall a matching version " + "({!r} installed, {!r} bundled)") + print(msg.format(pip.__version__, _PIP_VERSION), file=sys.stderr) + return + + _require_ssl_for_pip() + _disable_pip_configuration_settings() + + # Construct the arguments to be passed to the pip command + args = ["uninstall", "-y"] + if verbosity: + args += ["-" + "v" * verbosity] + + _run_pip(args + [p[0] for p in reversed(_PROJECTS)]) + + +def _main(argv=None): + if ssl is None: + print("Ignoring ensurepip failure: {}".format(_MISSING_SSL_MESSAGE), + file=sys.stderr) + return + + import argparse + parser = argparse.ArgumentParser(prog="python -m ensurepip") + parser.add_argument( + "--version", + action="version", + version="pip {}".format(version()), + help="Show the version of pip that is bundled with this Python.", + ) + parser.add_argument( + "-v", "--verbose", + action="count", + default=0, + dest="verbosity", + help=("Give more output. Option is additive, and can be used up to 3 " + "times."), + ) + parser.add_argument( + "-U", "--upgrade", + action="store_true", + default=False, + help="Upgrade pip and dependencies, even if already installed.", + ) + parser.add_argument( + "--user", + action="store_true", + default=False, + help="Install using the user scheme.", + ) + parser.add_argument( + "--root", + default=None, + help="Install everything relative to this alternate root directory.", + ) + parser.add_argument( + "--altinstall", + action="store_true", + default=False, + help=("Make an alternate install, installing only the X.Y versioned" + "scripts (Default: pipX, pipX.Y, easy_install-X.Y)"), + ) + parser.add_argument( + "--default-pip", + action="store_true", + default=True, + dest="default_pip", + help=argparse.SUPPRESS, + ) + parser.add_argument( + "--no-default-pip", + action="store_false", + dest="default_pip", + help=("Make a non default install, installing only the X and X.Y " + "versioned scripts."), + ) + + args = parser.parse_args(argv) + + bootstrap( + root=args.root, + upgrade=args.upgrade, + user=args.user, + verbosity=args.verbosity, + altinstall=args.altinstall, + default_pip=args.default_pip, + ) diff --git a/lib-python/2.7/ensurepip/__main__.py b/lib-python/2.7/ensurepip/__main__.py new file mode 100644 --- /dev/null +++ b/lib-python/2.7/ensurepip/__main__.py @@ -0,0 +1,4 @@ +import ensurepip + +if __name__ == "__main__": + ensurepip._main() diff --git a/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..097ab43430d4c1302b0be353a8c16407c370693b GIT binary patch [cut] diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..fa1d1054da1dab98f8906555d31a9fda271b3a85 GIT binary patch [cut] diff --git a/lib-python/2.7/ensurepip/_uninstall.py b/lib-python/2.7/ensurepip/_uninstall.py new file mode 100644 --- /dev/null +++ b/lib-python/2.7/ensurepip/_uninstall.py @@ -0,0 +1,30 @@ +"""Basic pip uninstallation support, helper for the Windows uninstaller""" + +import argparse +import ensurepip + + +def _main(argv=None): + parser = argparse.ArgumentParser(prog="python -m ensurepip._uninstall") + parser.add_argument( + "--version", + action="version", + version="pip {}".format(ensurepip.version()), + help="Show the version of pip this will attempt to uninstall.", + ) + parser.add_argument( + "-v", "--verbose", + action="count", + default=0, + dest="verbosity", + help=("Give more output. Option is additive, and can be used up to 3 " + "times."), + ) + + args = parser.parse_args(argv) + + ensurepip._uninstall_helper(verbosity=args.verbosity) + + +if __name__ == "__main__": + _main() diff --git a/lib-python/2.7/glob.py b/lib-python/2.7/glob.py --- a/lib-python/2.7/glob.py +++ b/lib-python/2.7/glob.py @@ -35,11 +35,16 @@ patterns. """ + dirname, basename = os.path.split(pathname) if not has_magic(pathname): - if os.path.lexists(pathname): - yield pathname + if basename: + if os.path.lexists(pathname): + yield pathname + else: + # Patterns ending with a slash should match only directories + if os.path.isdir(dirname): + yield pathname return - dirname, basename = os.path.split(pathname) if not dirname: for name in glob1(os.curdir, basename): yield name diff --git a/lib-python/2.7/gzip.py b/lib-python/2.7/gzip.py --- a/lib-python/2.7/gzip.py +++ b/lib-python/2.7/gzip.py @@ -164,9 +164,16 @@ def _write_gzip_header(self): self.fileobj.write('\037\213') # magic header self.fileobj.write('\010') # compression method - fname = os.path.basename(self.name) - if fname.endswith(".gz"): - fname = fname[:-3] + try: + # RFC 1952 requires the FNAME field to be Latin-1. Do not + # include filenames that cannot be represented that way. + fname = os.path.basename(self.name) + if not isinstance(fname, str): + fname = fname.encode('latin-1') + if fname.endswith('.gz'): + fname = fname[:-3] + except UnicodeEncodeError: + fname = '' flags = 0 if fname: flags = FNAME diff --git a/lib-python/2.7/hashlib.py b/lib-python/2.7/hashlib.py --- a/lib-python/2.7/hashlib.py +++ b/lib-python/2.7/hashlib.py @@ -15,8 +15,9 @@ md5(), sha1(), sha224(), sha256(), sha384(), and sha512() -More algorithms may be available on your platform but the above are -guaranteed to exist. +More algorithms may be available on your platform but the above are guaranteed +to exist. See the algorithms_guaranteed and algorithms_available attributes +to find out what algorithm names can be passed to new(). NOTE: If you want the adler32 or crc32 hash functions they are available in the zlib module. @@ -58,9 +59,14 @@ # always available algorithm is added. __always_supported = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512') +algorithms_guaranteed = set(__always_supported) +algorithms_available = set(__always_supported) + algorithms = __always_supported -__all__ = __always_supported + ('new', 'algorithms', 'pbkdf2_hmac') +__all__ = __always_supported + ('new', 'algorithms_guaranteed', + 'algorithms_available', 'algorithms', + 'pbkdf2_hmac') def __get_builtin_constructor(name): @@ -128,6 +134,8 @@ import _hashlib new = __hash_new __get_hash = __get_openssl_constructor + algorithms_available = algorithms_available.union( + _hashlib.openssl_md_meth_names) except ImportError: new = __py_new __get_hash = __get_builtin_constructor diff --git a/lib-python/2.7/httplib.py b/lib-python/2.7/httplib.py --- a/lib-python/2.7/httplib.py +++ b/lib-python/2.7/httplib.py @@ -215,6 +215,10 @@ # maximal line length when calling readline(). _MAXLINE = 65536 +# maximum amount of headers accepted +_MAXHEADERS = 100 + + class HTTPMessage(mimetools.Message): def addheader(self, key, value): @@ -271,6 +275,8 @@ elif self.seekable: tell = self.fp.tell while True: + if len(hlist) > _MAXHEADERS: + raise HTTPException("got more than %d headers" % _MAXHEADERS) if tell: try: startofline = tell() @@ -1185,21 +1191,29 @@ def __init__(self, host, port=None, key_file=None, cert_file=None, strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - source_address=None): + source_address=None, context=None): HTTPConnection.__init__(self, host, port, strict, timeout, source_address) self.key_file = key_file self.cert_file = cert_file + if context is None: + context = ssl._create_default_https_context() + if key_file or cert_file: + context.load_cert_chain(cert_file, key_file) + self._context = context def connect(self): "Connect to a host on a given (SSL) port." - sock = self._create_connection((self.host, self.port), - self.timeout, self.source_address) + HTTPConnection.connect(self) + if self._tunnel_host: - self.sock = sock - self._tunnel() - self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file) + server_hostname = self._tunnel_host + else: + server_hostname = self.host + + self.sock = self._context.wrap_socket(self.sock, + server_hostname=server_hostname) __all__.append("HTTPSConnection") @@ -1214,14 +1228,15 @@ _connection_class = HTTPSConnection def __init__(self, host='', port=None, key_file=None, cert_file=None, - strict=None): + strict=None, context=None): # provide a default host, pass the X509 cert info # urf. compensate for bad input. if port == 0: port = None self._setup(self._connection_class(host, port, key_file, - cert_file, strict)) + cert_file, strict, + context=context)) # we never actually use these for anything, but we keep them # here for compatibility with post-1.5.2 CVS. diff --git a/lib-python/2.7/idlelib/Bindings.py b/lib-python/2.7/idlelib/Bindings.py --- a/lib-python/2.7/idlelib/Bindings.py +++ b/lib-python/2.7/idlelib/Bindings.py @@ -75,7 +75,8 @@ ('!_Auto-open Stack Viewer', '<>'), ]), ('options', [ - ('_Configure IDLE...', '<>'), + ('Configure _IDLE', '<>'), + ('Configure _Extensions', '<>'), None, ]), ('help', [ diff --git a/lib-python/2.7/idlelib/CallTipWindow.py b/lib-python/2.7/idlelib/CallTipWindow.py --- a/lib-python/2.7/idlelib/CallTipWindow.py +++ b/lib-python/2.7/idlelib/CallTipWindow.py @@ -2,9 +2,8 @@ After ToolTip.py, which uses ideas gleaned from PySol Used by the CallTips IDLE extension. - """ -from Tkinter import * +from Tkinter import Toplevel, Label, LEFT, SOLID, TclError HIDE_VIRTUAL_EVENT_NAME = "<>" HIDE_SEQUENCES = ("", "") @@ -133,35 +132,28 @@ return bool(self.tipwindow) -def _calltip_window(parent): - root = Tk() - root.title("Test calltips") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) +def _calltip_window(parent): # htest # + from Tkinter import Toplevel, Text, LEFT, BOTH - class MyEditWin: # comparenceptually an editor_window - def __init__(self): - text = self.text = Text(root) - text.pack(side=LEFT, fill=BOTH, expand=1) - text.insert("insert", "string.split") - root.update() - self.calltip = CallTip(text) + top = Toplevel(parent) + top.title("Test calltips") + top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200, + parent.winfo_rooty() + 150)) + text = Text(top) + text.pack(side=LEFT, fill=BOTH, expand=1) + text.insert("insert", "string.split") + top.update() + calltip = CallTip(text) - text.event_add("<>", "(") - text.event_add("<>", ")") - text.bind("<>", self.calltip_show) - text.bind("<>", self.calltip_hide) - - text.focus_set() - root.mainloop() - - def calltip_show(self, event): - self.calltip.showtip("Hello world", "insert", "end") - - def calltip_hide(self, event): - self.calltip.hidetip() - - editwin = MyEditWin() + def calltip_show(event): + calltip.showtip("(s=Hello world)", "insert", "end") + def calltip_hide(event): + calltip.hidetip() + text.event_add("<>", "(") + text.event_add("<>", ")") + text.bind("<>", calltip_show) + text.bind("<>", calltip_hide) + text.focus_set() if __name__=='__main__': from idlelib.idle_test.htest import run diff --git a/lib-python/2.7/idlelib/ClassBrowser.py b/lib-python/2.7/idlelib/ClassBrowser.py --- a/lib-python/2.7/idlelib/ClassBrowser.py +++ b/lib-python/2.7/idlelib/ClassBrowser.py @@ -19,6 +19,9 @@ from idlelib.TreeWidget import TreeNode, TreeItem, ScrolledCanvas from idlelib.configHandler import idleConf +file_open = None # Method...Item and Class...Item use this. +# Normally PyShell.flist.open, but there is no PyShell.flist for htest. + class ClassBrowser: def __init__(self, flist, name, path, _htest=False): @@ -27,6 +30,9 @@ """ _htest - bool, change box when location running htest. """ + global file_open + if not _htest: + file_open = PyShell.flist.open self.name = name self.file = os.path.join(path[0], self.name + ".py") self._htest = _htest @@ -101,7 +107,7 @@ return [] try: dict = pyclbr.readmodule_ex(name, [dir] + sys.path) - except ImportError, msg: + except ImportError: return [] items = [] self.classes = {} @@ -170,7 +176,7 @@ def OnDoubleClick(self): if not os.path.exists(self.file): return - edit = PyShell.flist.open(self.file) + edit = file_open(self.file) if hasattr(self.cl, 'lineno'): lineno = self.cl.lineno edit.gotoline(lineno) @@ -206,7 +212,7 @@ def OnDoubleClick(self): if not os.path.exists(self.file): return - edit = PyShell.flist.open(self.file) + edit = file_open(self.file) edit.gotoline(self.cl.methods[self.name]) def _class_browser(parent): #Wrapper for htest @@ -221,8 +227,9 @@ dir, file = os.path.split(file) name = os.path.splitext(file)[0] flist = PyShell.PyShellFileList(parent) + global file_open + file_open = flist.open ClassBrowser(flist, name, [dir], _htest=True) - parent.mainloop() if __name__ == "__main__": from idlelib.idle_test.htest import run diff --git a/lib-python/2.7/idlelib/ColorDelegator.py b/lib-python/2.7/idlelib/ColorDelegator.py --- a/lib-python/2.7/idlelib/ColorDelegator.py +++ b/lib-python/2.7/idlelib/ColorDelegator.py @@ -2,7 +2,6 @@ import re import keyword import __builtin__ -from Tkinter import * from idlelib.Delegator import Delegator from idlelib.configHandler import idleConf @@ -34,7 +33,6 @@ prog = re.compile(make_pat(), re.S) idprog = re.compile(r"\s+(\w+)", re.S) -asprog = re.compile(r".*?\b(as)\b") class ColorDelegator(Delegator): @@ -42,7 +40,6 @@ Delegator.__init__(self) self.prog = prog self.idprog = idprog - self.asprog = asprog self.LoadTagDefs() def setdelegate(self, delegate): @@ -74,7 +71,6 @@ "DEFINITION": idleConf.GetHighlight(theme, "definition"), "SYNC": {'background':None,'foreground':None}, "TODO": {'background':None,'foreground':None}, - "BREAK": idleConf.GetHighlight(theme, "break"), "ERROR": idleConf.GetHighlight(theme, "error"), # The following is used by ReplaceDialog: "hit": idleConf.GetHighlight(theme, "hit"), @@ -216,22 +212,6 @@ self.tag_add("DEFINITION", head + "+%dc" % a, head + "+%dc" % b) - elif value == "import": - # color all the "as" words on same line, except - # if in a comment; cheap approximation to the - # truth - if '#' in chars: - endpos = chars.index('#') - else: - endpos = len(chars) - while True: - m1 = self.asprog.match(chars, b, endpos) - if not m1: - break - a, b = m1.span(1) - self.tag_add("KEYWORD", - head + "+%dc" % a, - head + "+%dc" % b) m = self.prog.search(chars, m.end()) if "SYNC" in self.tag_names(next + "-1c"): head = next @@ -255,20 +235,23 @@ for tag in self.tagdefs.keys(): self.tag_remove(tag, "1.0", "end") -def _color_delegator(parent): +def _color_delegator(parent): # htest # + from Tkinter import Toplevel, Text from idlelib.Percolator import Percolator - root = Tk() - root.title("Test ColorDelegator") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) - source = "if somename: x = 'abc' # comment\nprint" - text = Text(root, background="white") + + top = Toplevel(parent) + top.title("Test ColorDelegator") + top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200, From noreply at buildbot.pypy.org Mon Apr 6 11:41:12 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 6 Apr 2015 11:41:12 +0200 (CEST) Subject: [pypy-commit] buildbot default: fix Message-ID: <20150406094112.7ED2C1C0334@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r945:09b640169fe5 Date: 2015-04-06 11:41 +0200 http://bitbucket.org/pypy/buildbot/changeset/09b640169fe5/ Log: fix diff --git a/README_BUILDSLAVE b/README_BUILDSLAVE --- a/README_BUILDSLAVE +++ b/README_BUILDSLAVE @@ -2,7 +2,7 @@ ================================== First you will need to install the ``buildbot-slave`` package. -pip install buildbot_buildslave +pip install buildbot-slave The next step is to create a buildslave configuration file. Based on version 0.8.7 of buildbot you need to execute the following command. From noreply at buildbot.pypy.org Mon Apr 6 11:56:49 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 6 Apr 2015 11:56:49 +0200 (CEST) Subject: [pypy-commit] buildbot default: Move from allegro64 to speed-old. Message-ID: <20150406095649.A2C3A1C0334@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r946:d3ffa88409f1 Date: 2015-04-06 11:57 +0200 http://bitbucket.org/pypy/buildbot/changeset/d3ffa88409f1/ Log: Move from allegro64 to speed-old. diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -254,17 +254,17 @@ 'change_source': [], 'schedulers': [ - # the benchmarks run on tannit and speed.python.org. - # All the other linux tests run on allegro + # the benchmarks run on tannit and (planned) speed-old.python.org. + # All the other linux tests run on speed-old.python.org. Nightly("nightly-0-00", [ # benchmarks # linux tests LINUX32, # on tannit32, uses all cores - LINUX64, # on allegro64, uses all cores + LINUX64, # on speed-old, uses all cores JITLINUX32, # on tannit32, uses 1 core - JITLINUX64, # on allegro64, uses 1 core + JITLINUX64, # on speed-old, uses 1 core #APPLVLLINUX32, # on tannit32, uses 1 core - APPLVLLINUX64, # on allegro64, uses 1 core + APPLVLLINUX64, # on speed-old, uses 1 core # other platforms #MACOSX32, # on minime JITWIN32, # on allegro_win32, SalsaSalsa @@ -293,8 +293,8 @@ ]), Nightly("nightly-2-00-py3k", [ - LINUX64, # on allegro64, uses all cores - APPLVLLINUX64, # on allegro64, uses 1 core + LINUX64, # on speed-old, uses all cores + APPLVLLINUX64, # on speed-old, uses 1 core ], branch="py3k", hour=2, minute=0), # this one has faithfully run every night even though the latest @@ -355,7 +355,7 @@ "locks": [TannitCPU.access('counting')], }, {"name": LINUX64, - "slavenames": ["allegro64"], + "slavenames": ["speed-old"], "builddir": LINUX64, "factory": pypyOwnTestFactory, "category": 'linux64', @@ -370,7 +370,7 @@ "locks": [TannitCPU.access('counting')], }, {"name": APPLVLLINUX64, - "slavenames": ["allegro64"], + "slavenames": ["speed-old"], "builddir": APPLVLLINUX64, "factory": pypyTranslatedAppLevelTestFactory64, "category": "linux64", @@ -385,7 +385,7 @@ "locks": [TannitCPU.access('counting')], }, {"name": LIBPYTHON_LINUX64, - "slavenames": ["allegro64"], + "slavenames": ["speed-old"], "builddir": LIBPYTHON_LINUX64, "factory": pypyTranslatedLibPythonTestFactory, "category": "linux64", @@ -400,7 +400,7 @@ "locks": [TannitCPU.access('counting')], }, {'name': JITLINUX64, - 'slavenames': ["allegro64"], + 'slavenames': ["speed-old"], 'builddir': JITLINUX64, 'factory': pypyJITTranslatedTestFactory64, 'category': 'linux64', From noreply at buildbot.pypy.org Mon Apr 6 13:35:47 2015 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 6 Apr 2015 13:35:47 +0200 (CEST) Subject: [pypy-commit] pypy vmprof: disable passing seconds Message-ID: <20150406113547.455FA1C14CA@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76720:d602c99eb58b Date: 2015-04-06 13:35 +0200 http://bitbucket.org/pypy/pypy/changeset/d602c99eb58b/ Log: disable passing seconds diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py --- a/pypy/module/_vmprof/interp_vmprof.py +++ b/pypy/module/_vmprof/interp_vmprof.py @@ -224,7 +224,8 @@ # try: period_usec = ovfcheck_float_to_int(period * 1000000.0 + 0.5) - if period_usec <= 0: + if period_usec <= 0 or period_usec >= 1e9: + # we don't want seconds here at all raise ValueError except (ValueError, OverflowError): raise OperationError(space.w_ValueError, From noreply at buildbot.pypy.org Mon Apr 6 13:58:31 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 6 Apr 2015 13:58:31 +0200 (CEST) Subject: [pypy-commit] pypy more-rposix: Move ll_os_stat to rposix_stat. Message-ID: <20150406115831.2E5F21C0204@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: more-rposix Changeset: r76721:8ff6cf17a905 Date: 2015-04-06 13:42 +0200 http://bitbucket.org/pypy/pypy/changeset/8ff6cf17a905/ Log: Move ll_os_stat to rposix_stat. Not tested at all on Windows. diff --git a/pypy/module/_io/interp_io.py b/pypy/module/_io/interp_io.py --- a/pypy/module/_io/interp_io.py +++ b/pypy/module/_io/interp_io.py @@ -7,7 +7,9 @@ from pypy.module.exceptions.interp_exceptions import W_IOError from pypy.module._io.interp_fileio import W_FileIO from pypy.module._io.interp_textio import W_TextIOWrapper -from rpython.rtyper.module.ll_os_stat import STAT_FIELD_TYPES +from rpython.rlib.rposix_stat import STAT_FIELD_TYPES + +HAS_BLKSIZE = 'st_blksize' in STAT_FIELD_TYPES class Cache: @@ -118,7 +120,7 @@ if buffering < 0: buffering = DEFAULT_BUFFER_SIZE - if space.config.translation.type_system == 'lltype' and 'st_blksize' in STAT_FIELD_TYPES: + if HAS_BLKSIZE: fileno = space.c_int_w(space.call_method(w_raw, "fileno")) try: st = os.fstat(fileno) diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py --- a/pypy/module/posix/interp_posix.py +++ b/pypy/module/posix/interp_posix.py @@ -1,11 +1,11 @@ import os import sys -from rpython.rlib import rposix, objectmodel, rurandom +from rpython.rlib import rposix, rposix_stat +from rpython.rlib import objectmodel, rurandom from rpython.rlib.objectmodel import specialize from rpython.rlib.rarithmetic import r_longlong from rpython.rlib.unroll import unrolling_iterable -from rpython.rtyper.module import ll_os_stat from pypy.interpreter.gateway import unwrap_spec from pypy.interpreter.error import OperationError, wrap_oserror, wrap_oserror2 @@ -215,13 +215,13 @@ # ____________________________________________________________ -STAT_FIELDS = unrolling_iterable(enumerate(ll_os_stat.STAT_FIELDS)) +STAT_FIELDS = unrolling_iterable(enumerate(rposix_stat.STAT_FIELDS)) -STATVFS_FIELDS = unrolling_iterable(enumerate(ll_os_stat.STATVFS_FIELDS)) +STATVFS_FIELDS = unrolling_iterable(enumerate(rposix_stat.STATVFS_FIELDS)) def build_stat_result(space, st): FIELDS = STAT_FIELDS # also when not translating at all - lst = [None] * ll_os_stat.N_INDEXABLE_FIELDS + lst = [None] * rposix_stat.N_INDEXABLE_FIELDS w_keywords = space.newdict() stat_float_times = space.fromcache(StatState).stat_float_times for i, (name, TYPE) in FIELDS: @@ -229,7 +229,7 @@ if name in ('st_atime', 'st_mtime', 'st_ctime'): value = int(value) # rounded to an integer for indexed access w_value = space.wrap(value) - if i < ll_os_stat.N_INDEXABLE_FIELDS: + if i < rposix_stat.N_INDEXABLE_FIELDS: lst[i] = w_value else: space.setitem(w_keywords, space.wrap(name), w_value) @@ -257,7 +257,7 @@ def build_statvfs_result(space, st): - vals_w = [None] * len(ll_os_stat.STATVFS_FIELDS) + vals_w = [None] * len(rposix_stat.STATVFS_FIELDS) for i, (name, _) in STATVFS_FIELDS: vals_w[i] = space.wrap(getattr(st, name)) w_tuple = space.newtuple(vals_w) @@ -270,7 +270,7 @@ """Perform a stat system call on the file referenced to by an open file descriptor.""" try: - st = os.fstat(fd) + st = rposix_stat.fstat(fd) except OSError, e: raise wrap_oserror(space, e) else: @@ -292,7 +292,7 @@ """ try: - st = dispatch_filename(rposix.stat)(space, w_path) + st = dispatch_filename(rposix_stat.stat)(space, w_path) except OSError, e: raise wrap_oserror2(space, e, w_path) else: @@ -301,7 +301,7 @@ def lstat(space, w_path): "Like stat(path), but do no follow symbolic links." try: - st = dispatch_filename(rposix.lstat)(space, w_path) + st = dispatch_filename(rposix_stat.lstat)(space, w_path) except OSError, e: raise wrap_oserror2(space, e, w_path) else: @@ -330,7 +330,7 @@ @unwrap_spec(fd=c_int) def fstatvfs(space, fd): try: - st = os.fstatvfs(fd) + st = rposix_stat.fstatvfs(fd) except OSError as e: raise wrap_oserror(space, e) else: @@ -339,7 +339,7 @@ def statvfs(space, w_path): try: - st = dispatch_filename(rposix.statvfs)(space, w_path) + st = dispatch_filename(rposix_stat.statvfs)(space, w_path) except OSError as e: raise wrap_oserror2(space, e, w_path) else: diff --git a/rpython/rlib/_rposix_repr.py b/rpython/rlib/_rposix_repr.py new file mode 100644 --- /dev/null +++ b/rpython/rlib/_rposix_repr.py @@ -0,0 +1,122 @@ +""" +RTyping support for os.stat_result objects. +They are rtyped just like a tuple of the correct length supporting +only indexing and the st_xxx attributes. We need a custom StatResultRepr +because when rtyping for LL backends we have extra platform-dependent +items at the end of the tuple, but for OO backends we only want the +portable items. This allows the OO backends to assume a fixed shape for +the tuples returned by os.stat(). +""" +from rpython.annotator import model as annmodel +from rpython.rtyper.llannotation import lltype_to_annotation +from rpython.flowspace.model import Constant +from rpython.flowspace.operation import op +from rpython.tool.pairtype import pairtype +from rpython.rtyper.rmodel import Repr +from rpython.rtyper.rint import IntegerRepr +from rpython.rtyper.error import TyperError +from rpython.rlib import rposix_stat + + +class StatResultRepr(Repr): + + def __init__(self, rtyper): + self.rtyper = rtyper + self.stat_fields = rposix_stat.STAT_FIELDS + + self.stat_field_indexes = {} + for i, (name, TYPE) in enumerate(self.stat_fields): + self.stat_field_indexes[name] = i + + self.s_tuple = annmodel.SomeTuple([lltype_to_annotation(TYPE) + for name, TYPE in self.stat_fields]) + self.r_tuple = rtyper.getrepr(self.s_tuple) + self.lowleveltype = self.r_tuple.lowleveltype + + def redispatch_getfield(self, hop, index): + rtyper = self.rtyper + s_index = rtyper.annotator.bookkeeper.immutablevalue(index) + hop2 = hop.copy() + spaceop = op.getitem(hop.args_v[0], Constant(index)) + spaceop.result = hop.spaceop.result + hop2.spaceop = spaceop + hop2.args_v = spaceop.args + hop2.args_s = [self.s_tuple, s_index] + hop2.args_r = [self.r_tuple, rtyper.getrepr(s_index)] + return hop2.dispatch() + + def rtype_getattr(self, hop): + s_attr = hop.args_s[1] + attr = s_attr.const + try: + index = self.stat_field_indexes[attr] + except KeyError: + raise TyperError("os.stat().%s: field not available" % (attr,)) + return self.redispatch_getfield(hop, index) + + +class __extend__(pairtype(StatResultRepr, IntegerRepr)): + + def rtype_getitem((r_sta, r_int), hop): + s_int = hop.args_s[1] + index = s_int.const + return r_sta.redispatch_getfield(hop, index) + + +def specialize_make_stat_result(hop): + r_StatResult = hop.rtyper.getrepr(rposix_stat.s_StatResult) + [v_result] = hop.inputargs(r_StatResult.r_tuple) + # no-op conversion from r_StatResult.r_tuple to r_StatResult + hop.exception_cannot_occur() + return v_result + + +class StatvfsResultRepr(Repr): + + def __init__(self, rtyper): + self.rtyper = rtyper + self.statvfs_fields = rposix_stat.STATVFS_FIELDS + + self.statvfs_field_indexes = {} + for i, (name, TYPE) in enumerate(self.statvfs_fields): + self.statvfs_field_indexes[name] = i + + self.s_tuple = annmodel.SomeTuple([lltype_to_annotation(TYPE) + for name, TYPE in self.statvfs_fields]) + self.r_tuple = rtyper.getrepr(self.s_tuple) + self.lowleveltype = self.r_tuple.lowleveltype + + def redispatch_getfield(self, hop, index): + rtyper = self.rtyper + s_index = rtyper.annotator.bookkeeper.immutablevalue(index) + hop2 = hop.copy() + spaceop = op.getitem(hop.args_v[0], Constant(index)) + spaceop.result = hop.spaceop.result + hop2.spaceop = spaceop + hop2.args_v = spaceop.args + hop2.args_s = [self.s_tuple, s_index] + hop2.args_r = [self.r_tuple, rtyper.getrepr(s_index)] + return hop2.dispatch() + + def rtype_getattr(self, hop): + s_attr = hop.args_s[1] + attr = s_attr.const + try: + index = self.statvfs_field_indexes[attr] + except KeyError: + raise TyperError("os.statvfs().%s: field not available" % (attr,)) + return self.redispatch_getfield(hop, index) + + +class __extend__(pairtype(StatvfsResultRepr, IntegerRepr)): + def rtype_getitem((r_sta, r_int), hop): + s_int = hop.args_s[1] + index = s_int.const + return r_sta.redispatch_getfield(hop, index) + + +def specialize_make_statvfs_result(hop): + r_StatvfsResult = hop.rtyper.getrepr(rposix_stat.s_StatvfsResult) + [v_result] = hop.inputargs(r_StatvfsResult.r_tuple) + hop.exception_cannot_occur() + return v_result diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py --- a/rpython/rlib/rarithmetic.py +++ b/rpython/rlib/rarithmetic.py @@ -534,7 +534,7 @@ else: r_int64 = int -# needed for ll_os_stat.time_t_to_FILE_TIME in the 64 bit case +# needed for rposix_stat.time_t_to_FILE_TIME in the 64 bit case r_uint32 = build_int('r_uint32', False, 32) SHRT_MIN = -2**(_get_bitsize('h') - 1) diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -355,18 +355,6 @@ def _preferred_traits(path): return string_traits - at specialize.argtype(0) -def stat(path): - return os.stat(_as_bytes(path)) - - at specialize.argtype(0) -def lstat(path): - return os.lstat(_as_bytes(path)) - - at specialize.argtype(0) -def statvfs(path): - return os.statvfs(_as_bytes(path)) - @specialize.argtype(0, 1) def putenv(name, value): os.environ[_as_bytes(name)] = _as_bytes(value) @@ -1284,7 +1272,7 @@ if not _WIN32: l_tmsbuf = lltype.malloc(TMSP.TO, flavor='raw') try: - handle_posix_error('times', c_times(l_tmsbuf)) + result = handle_posix_error('times', c_times(l_tmsbuf)) return ( rffi.cast(lltype.Signed, l_tmsbuf.c_tms_utime) / CLOCK_TICKS_PER_SECOND, diff --git a/rpython/rlib/rposix_stat.py b/rpython/rlib/rposix_stat.py new file mode 100644 --- /dev/null +++ b/rpython/rlib/rposix_stat.py @@ -0,0 +1,523 @@ +"""Annotation and rtyping support for the result of os.stat(), os.lstat() +and os.fstat(). In RPython like in plain Python the stat result can be +indexed like a tuple but also exposes the st_xxx attributes. +""" + +import os, sys + +from rpython.annotator import model as annmodel +from rpython.rtyper import extregistry +from rpython.tool.pairtype import pairtype +from rpython.rtyper.tool import rffi_platform as platform +from rpython.rtyper.llannotation import lltype_to_annotation + +from rpython.rlib.objectmodel import ( + specialize) +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.translator.tool.cbuild import ExternalCompilationInfo +from rpython.rlib.rposix import ( + replace_os_function, handle_posix_error, _as_bytes0) + +_WIN32 = sys.platform.startswith('win') +_LINUX = sys.platform.startswith('linux') + +# Support for float times is here. +# - ALL_STAT_FIELDS contains Float fields if the system can retrieve +# sub-second timestamps. +# - TIMESPEC is defined when the "struct stat" contains st_atim field. + +if sys.platform.startswith('linux') or sys.platform.startswith('openbsd'): + TIMESPEC = platform.Struct('struct timespec', + [('tv_sec', rffi.TIME_T), + ('tv_nsec', rffi.LONG)]) +else: + TIMESPEC = None + +# all possible fields - some of them are not available on all platforms +ALL_STAT_FIELDS = [ + ("st_mode", lltype.Signed), + ("st_ino", lltype.SignedLongLong), + ("st_dev", lltype.SignedLongLong), + ("st_nlink", lltype.Signed), + ("st_uid", lltype.Signed), + ("st_gid", lltype.Signed), + ("st_size", lltype.SignedLongLong), + ("st_atime", lltype.Float), + ("st_mtime", lltype.Float), + ("st_ctime", lltype.Float), + ("st_blksize", lltype.Signed), + ("st_blocks", lltype.Signed), + ("st_rdev", lltype.Signed), + ("st_flags", lltype.Signed), + #("st_gen", lltype.Signed), -- new in CPy 2.5, not implemented + #("st_birthtime", lltype.Float), -- new in CPy 2.5, not implemented +] +N_INDEXABLE_FIELDS = 10 + +# For OO backends, expose only the portable fields (the first 10). +PORTABLE_STAT_FIELDS = ALL_STAT_FIELDS[:N_INDEXABLE_FIELDS] + +STATVFS_FIELDS = [ + ("f_bsize", lltype.Signed), + ("f_frsize", lltype.Signed), + ("f_blocks", lltype.Signed), + ("f_bfree", lltype.Signed), + ("f_bavail", lltype.Signed), + ("f_files", lltype.Signed), + ("f_ffree", lltype.Signed), + ("f_favail", lltype.Signed), + ("f_flag", lltype.Signed), + ("f_namemax", lltype.Signed), +] + + +# ____________________________________________________________ +# +# Annotation support + +class SomeStatResult(annmodel.SomeObject): + knowntype = os.stat_result + + def rtyper_makerepr(self, rtyper): + from rpython.rlib import _rposix_repr + return _rposix_repr.StatResultRepr(rtyper) + + def rtyper_makekey(self): + return self.__class__, + + def getattr(self, s_attr): + assert s_attr.is_constant(), "non-constant attr name in getattr()" + attrname = s_attr.const + TYPE = STAT_FIELD_TYPES[attrname] + return lltype_to_annotation(TYPE) + + def _get_rmarshall_support_(self): # for rlib.rmarshal + # reduce and recreate stat_result objects from 10-tuples + # (we ignore the extra values here for simplicity and portability) + def stat_result_reduce(st): + return (st[0], st[1], st[2], st[3], st[4], + st[5], st[6], st[7], st[8], st[9]) + + def stat_result_recreate(tup): + return make_stat_result(tup + extra_zeroes) + s_reduced = annmodel.SomeTuple([lltype_to_annotation(TYPE) + for name, TYPE in PORTABLE_STAT_FIELDS]) + extra_zeroes = (0,) * (len(STAT_FIELDS) - len(PORTABLE_STAT_FIELDS)) + return s_reduced, stat_result_reduce, stat_result_recreate + + +class SomeStatvfsResult(annmodel.SomeObject): + if hasattr(os, 'statvfs_result'): + knowntype = os.statvfs_result + else: + knowntype = None # will not be used + + def rtyper_makerepr(self, rtyper): + from rpython.rlib import _rposix_repr + return _rposix_repr.StatvfsResultRepr(rtyper) + + def rtyper_makekey(self): + return self.__class__, + + def getattr(self, s_attr): + assert s_attr.is_constant() + TYPE = STATVFS_FIELD_TYPES[s_attr.const] + return lltype_to_annotation(TYPE) + + +class __extend__(pairtype(SomeStatResult, annmodel.SomeInteger)): + def getitem((s_sta, s_int)): + assert s_int.is_constant(), "os.stat()[index]: index must be constant" + index = s_int.const + assert 0 <= index < N_INDEXABLE_FIELDS, "os.stat()[index] out of range" + name, TYPE = STAT_FIELDS[index] + return lltype_to_annotation(TYPE) + + +class __extend__(pairtype(SomeStatvfsResult, annmodel.SomeInteger)): + def getitem((s_stat, s_int)): + assert s_int.is_constant() + name, TYPE = STATVFS_FIELDS[s_int.const] + return lltype_to_annotation(TYPE) + + +s_StatResult = SomeStatResult() +s_StatvfsResult = SomeStatvfsResult() + + +def make_stat_result(tup): + """Turn a tuple into an os.stat_result object.""" + positional = tup[:N_INDEXABLE_FIELDS] + kwds = {} + for i, name in enumerate(STAT_FIELD_NAMES[N_INDEXABLE_FIELDS:]): + kwds[name] = tup[N_INDEXABLE_FIELDS + i] + return os.stat_result(positional, kwds) + + +def make_statvfs_result(tup): + return os.statvfs_result(tup) + + +class MakeStatResultEntry(extregistry.ExtRegistryEntry): + _about_ = make_stat_result + + def compute_result_annotation(self, s_tup): + return s_StatResult + + def specialize_call(self, hop): + from rpython.rlib import _rposix_repr + return _rposix_repr.specialize_make_stat_result(hop) + + +class MakeStatvfsResultEntry(extregistry.ExtRegistryEntry): + _about_ = make_statvfs_result + + def compute_result_annotation(self, s_tup): + return s_StatvfsResult + + def specialize_call(self, hop): + from rpython.rlib import _rposix_repr + return _rposix_repr.specialize_make_statvfs_result(hop) + +# ____________________________________________________________ +# +# RFFI support + +if sys.platform.startswith('win'): + _name_struct_stat = '_stati64' + INCLUDES = ['sys/types.h', 'sys/stat.h', 'sys/statvfs.h'] +else: + if _LINUX: + _name_struct_stat = 'stat64' + else: + _name_struct_stat = 'stat' + INCLUDES = ['sys/types.h', 'sys/stat.h', 'sys/statvfs.h', 'unistd.h'] + +compilation_info = ExternalCompilationInfo( + # This must be set to 64 on some systems to enable large file support. + #pre_include_bits = ['#define _FILE_OFFSET_BITS 64'], + # ^^^ nowadays it's always set in all C files we produce. + includes=INCLUDES +) + +if TIMESPEC is not None: + class CConfig_for_timespec: + _compilation_info_ = compilation_info + TIMESPEC = TIMESPEC + TIMESPEC = lltype.Ptr( + platform.configure(CConfig_for_timespec)['TIMESPEC']) + + +def posix_declaration(try_to_add=None): + global STAT_STRUCT, STATVFS_STRUCT + + LL_STAT_FIELDS = STAT_FIELDS[:] + if try_to_add: + LL_STAT_FIELDS.append(try_to_add) + + if TIMESPEC is not None: + + def _expand(lst, originalname, timespecname): + for i, (_name, _TYPE) in enumerate(lst): + if _name == originalname: + # replace the 'st_atime' field of type rffi.DOUBLE + # with a field 'st_atim' of type 'struct timespec' + lst[i] = (timespecname, TIMESPEC.TO) + break + + _expand(LL_STAT_FIELDS, 'st_atime', 'st_atim') + _expand(LL_STAT_FIELDS, 'st_mtime', 'st_mtim') + _expand(LL_STAT_FIELDS, 'st_ctime', 'st_ctim') + + del _expand + else: + # Replace float fields with integers + for name in ('st_atime', 'st_mtime', 'st_ctime', 'st_birthtime'): + for i, (_name, _TYPE) in enumerate(LL_STAT_FIELDS): + if _name == name: + LL_STAT_FIELDS[i] = (_name, lltype.Signed) + break + + class CConfig: + _compilation_info_ = compilation_info + STAT_STRUCT = platform.Struct('struct %s' % _name_struct_stat, LL_STAT_FIELDS) + STATVFS_STRUCT = platform.Struct('struct statvfs', STATVFS_FIELDS) + + try: + config = platform.configure(CConfig, ignore_errors=try_to_add is not None) + except platform.CompilationError: + if try_to_add: + return # failed to add this field, give up + raise + + STAT_STRUCT = lltype.Ptr(config['STAT_STRUCT']) + STATVFS_STRUCT = lltype.Ptr(config['STATVFS_STRUCT']) + if try_to_add: + STAT_FIELDS.append(try_to_add) + + +# This lists only the fields that have been found on the underlying platform. +# Initially only the PORTABLE_STAT_FIELDS, but more may be added by the +# following loop. +STAT_FIELDS = PORTABLE_STAT_FIELDS[:] + +if sys.platform != 'win32': + posix_declaration() + for _i in range(len(PORTABLE_STAT_FIELDS), len(ALL_STAT_FIELDS)): + posix_declaration(ALL_STAT_FIELDS[_i]) + del _i + +# these two global vars only list the fields defined in the underlying platform +STAT_FIELD_TYPES = dict(STAT_FIELDS) # {'st_xxx': TYPE} +STAT_FIELD_NAMES = [_name for (_name, _TYPE) in STAT_FIELDS] +del _name, _TYPE + +STATVFS_FIELD_TYPES = dict(STATVFS_FIELDS) +STATVFS_FIELD_NAMES = [name for name, tp in STATVFS_FIELDS] + +def build_stat_result(st): + # only for LL backends + if TIMESPEC is not None: + atim = st.c_st_atim; atime = int(atim.c_tv_sec) + 1E-9 * int(atim.c_tv_nsec) + mtim = st.c_st_mtim; mtime = int(mtim.c_tv_sec) + 1E-9 * int(mtim.c_tv_nsec) + ctim = st.c_st_ctim; ctime = int(ctim.c_tv_sec) + 1E-9 * int(ctim.c_tv_nsec) + else: + atime = st.c_st_atime + mtime = st.c_st_mtime + ctime = st.c_st_ctime + + result = (st.c_st_mode, + st.c_st_ino, + st.c_st_dev, + st.c_st_nlink, + st.c_st_uid, + st.c_st_gid, + st.c_st_size, + atime, + mtime, + ctime) + + if "st_blksize" in STAT_FIELD_TYPES: result += (st.c_st_blksize,) + if "st_blocks" in STAT_FIELD_TYPES: result += (st.c_st_blocks,) + if "st_rdev" in STAT_FIELD_TYPES: result += (st.c_st_rdev,) + if "st_flags" in STAT_FIELD_TYPES: result += (st.c_st_flags,) + + return make_stat_result(result) + + +def build_statvfs_result(st): + return make_statvfs_result(( + st.c_f_bsize, + st.c_f_frsize, + st.c_f_blocks, + st.c_f_bfree, + st.c_f_bavail, + st.c_f_files, + st.c_f_ffree, + st.c_f_favail, + st.c_f_flag, + st.c_f_namemax + )) + + +# Implement and register os.stat() & variants + +c_fstat = rffi.llexternal('fstat64' if _LINUX else 'fstat', + [rffi.INT, STAT_STRUCT], rffi.INT, + compilation_info=compilation_info, + save_err=rffi.RFFI_SAVE_ERRNO, + macro=True) +c_stat = rffi.llexternal('stat64' if _LINUX else 'stat', + [rffi.CCHARP, STAT_STRUCT], rffi.INT, + compilation_info=compilation_info, + save_err=rffi.RFFI_SAVE_ERRNO, + macro=True) +c_lstat = rffi.llexternal('lstat64' if _LINUX else 'lstat', + [rffi.CCHARP, STAT_STRUCT], rffi.INT, + compilation_info=compilation_info, + save_err=rffi.RFFI_SAVE_ERRNO, + macro=True) + +c_fstatvfs = rffi.llexternal('fstatvfs', + [rffi.INT, STATVFS_STRUCT], rffi.INT, + compilation_info=compilation_info, + save_err=rffi.RFFI_SAVE_ERRNO) +c_statvfs = rffi.llexternal('statvfs', + [rffi.CCHARP, STATVFS_STRUCT], rffi.INT, + compilation_info=compilation_info, + save_err=rffi.RFFI_SAVE_ERRNO) + + at replace_os_function('fstat') +def fstat(fd): + with lltype.scoped_alloc(STAT_STRUCT.TO) as stresult: + if not _WIN32: + handle_posix_error('fstat', c_fstat(fd, stresult)) + return build_stat_result(stresult) + else: + handle = rwin32.get_osfhandle(fd) + filetype = win32traits.GetFileType(handle) + if filetype == win32traits.FILE_TYPE_CHAR: + # console or LPT device + return make_stat_result((win32traits._S_IFCHR, + 0, 0, 0, 0, 0, + 0, 0, 0, 0)) + elif filetype == win32traits.FILE_TYPE_PIPE: + # socket or named pipe + return make_stat_result((win32traits._S_IFIFO, + 0, 0, 0, 0, 0, + 0, 0, 0, 0)) + elif filetype == win32traits.FILE_TYPE_UNKNOWN: + error = rwin32.GetLastError_saved() + if error != 0: + raise WindowsError(error, "os_fstat failed") + # else: unknown but valid file + + # normal disk file (FILE_TYPE_DISK) + info = lltype.malloc(win32traits.BY_HANDLE_FILE_INFORMATION, + flavor='raw', zero=True) + try: + res = win32traits.GetFileInformationByHandle(handle, info) + if res == 0: + raise WindowsError(rwin32.GetLastError_saved(), + "os_fstat failed") + return win32_by_handle_info_to_stat(info) + finally: + lltype.free(info, flavor='raw') + + at replace_os_function('stat') + at specialize.argtype(0) +def stat(path): + with lltype.scoped_alloc(STAT_STRUCT.TO) as stresult: + if not _WIN32: + arg = _as_bytes0(path) + handle_posix_error('stat', c_stat(arg, stresult)) + return build_stat_result(stresult) + else: + traits = _preferred_traits(path) + path = traits._as_str0(path) + return win32_xstat(path, traverse=True) + + at replace_os_function('lstat') + at specialize.argtype(0) +def lstat(path): + with lltype.scoped_alloc(STAT_STRUCT.TO) as stresult: + if not _WIN32: + arg = _as_bytes0(path) + handle_posix_error('lstat', c_stat(arg, stresult)) + else: + traits = _preferred_traits(path) + arg = traits._as_str0(path) + handle_posix_error('lstat', c_stat(arg, stresult)) + return build_stat_result(stresult) + + at replace_os_function('fstatvfs') +def fstatvfs(fd): + with lltype.scoped_alloc(STATVFS_STRUCT.TO) as stresult: + handle_posix_error('fstatvfs', c_fstatvfs(fd, stresult)) + return build_statvfs_result(stresult) + + at replace_os_function('statvfs') +def statvfs(path): + with lltype.scoped_alloc(STATVFS_STRUCT.TO) as stresult: + arg = _as_bytes0(path) + handle_posix_error('statvfs', c_statvfs(arg, stresult)) + return build_statvfs_result(stresult) + +#__________________________________________________ +# Helper functions for win32 +if _WIN32: + + def make_longlong(high, low): + return (rffi.r_longlong(high) << 32) + rffi.r_longlong(low) + + # Seconds between 1.1.1601 and 1.1.1970 + secs_between_epochs = rffi.r_longlong(11644473600) + + def FILE_TIME_to_time_t_float(filetime): + ft = make_longlong(filetime.c_dwHighDateTime, filetime.c_dwLowDateTime) + # FILETIME is in units of 100 nsec + return float(ft) * (1.0 / 10000000.0) - secs_between_epochs + + def time_t_to_FILE_TIME(time, filetime): + ft = rffi.r_longlong((time + secs_between_epochs) * 10000000) + filetime.c_dwHighDateTime = rffi.r_uint(ft >> 32) + filetime.c_dwLowDateTime = rffi.r_uint(ft) # masking off high bits + + def win32_xstat(traits, path, traverse=False): + win32traits = make_win32_traits(traits) + with lltype.scoped_alloc( + win32traits.WIN32_FILE_ATTRIBUTE_DATA) as data: + res = win32traits.GetFileAttributesEx( + path, win32traits.GetFileExInfoStandard, data) + if res == 0: + errcode = rwin32.GetLastError_saved() + if errcode == win32traits.ERROR_SHARING_VIOLATION: + res = win32_attributes_from_dir(path, data) + if res == 0: + errcode = rwin32.GetLastError_saved() + raise WindowsError(errcode, "os_stat failed") + return attribute_data_to_stat(data) + + def win32_attributes_to_mode(attributes): + m = 0 + attributes = intmask(attributes) + if attributes & win32traits.FILE_ATTRIBUTE_DIRECTORY: + m |= win32traits._S_IFDIR | 0111 # IFEXEC for user,group,other + else: + m |= win32traits._S_IFREG + if attributes & win32traits.FILE_ATTRIBUTE_READONLY: + m |= 0444 + else: + m |= 0666 + return m + + def win32_attribute_data_to_stat(info): + st_mode = win32_attributes_to_mode(info.c_dwFileAttributes) + st_size = make_longlong(info.c_nFileSizeHigh, info.c_nFileSizeLow) + ctime = FILE_TIME_to_time_t_float(info.c_ftCreationTime) + mtime = FILE_TIME_to_time_t_float(info.c_ftLastWriteTime) + atime = FILE_TIME_to_time_t_float(info.c_ftLastAccessTime) + + result = (st_mode, + 0, 0, 0, 0, 0, + st_size, + atime, mtime, ctime) + + return make_stat_result(result) + + def win32_by_handle_info_to_stat(info): + # similar to the one above + st_mode = win32_attributes_to_mode(info.c_dwFileAttributes) + st_size = make_longlong(info.c_nFileSizeHigh, info.c_nFileSizeLow) + ctime = FILE_TIME_to_time_t_float(info.c_ftCreationTime) + mtime = FILE_TIME_to_time_t_float(info.c_ftLastWriteTime) + atime = FILE_TIME_to_time_t_float(info.c_ftLastAccessTime) + + # specific to fstat() + st_ino = make_longlong(info.c_nFileIndexHigh, info.c_nFileIndexLow) + st_nlink = info.c_nNumberOfLinks + + result = (st_mode, + st_ino, 0, st_nlink, 0, 0, + st_size, + atime, mtime, ctime) + + return make_stat_result(result) + + def win32_attributes_from_dir(l_path, data): + filedata = lltype.malloc(win32traits.WIN32_FIND_DATA, flavor='raw') + try: + hFindFile = win32traits.FindFirstFile(l_path, filedata) + if hFindFile == rwin32.INVALID_HANDLE_VALUE: + return 0 + win32traits.FindClose(hFindFile) + data.c_dwFileAttributes = filedata.c_dwFileAttributes + rffi.structcopy(data.c_ftCreationTime, filedata.c_ftCreationTime) + rffi.structcopy(data.c_ftLastAccessTime, filedata.c_ftLastAccessTime) + rffi.structcopy(data.c_ftLastWriteTime, filedata.c_ftLastWriteTime) + data.c_nFileSizeHigh = filedata.c_nFileSizeHigh + data.c_nFileSizeLow = filedata.c_nFileSizeLow + return 1 + finally: + lltype.free(filedata, flavor='raw') + diff --git a/rpython/rlib/test/test_rmarshal.py b/rpython/rlib/test/test_rmarshal.py --- a/rpython/rlib/test/test_rmarshal.py +++ b/rpython/rlib/test/test_rmarshal.py @@ -167,7 +167,7 @@ def test_stat_result(): import os from rpython.translator.c.test.test_genc import compile - from rpython.rtyper.module.ll_os_stat import s_StatResult + from rpython.rlib.rposix_stat import s_StatResult marshal_stat_result = get_marshaller(s_StatResult) unmarshal_stat_result = get_unmarshaller(s_StatResult) def f(path): diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py --- a/rpython/rlib/test/test_rposix.py +++ b/rpython/rlib/test/test_rposix.py @@ -1,7 +1,7 @@ from rpython.rtyper.test.test_llinterp import interpret from rpython.translator.c.test.test_genc import compile from rpython.tool.udir import udir -from rpython.rlib import rposix, rstring +from rpython.rlib import rposix, rposix_stat, rstring import os, sys import py @@ -60,7 +60,7 @@ def test_stat(self): def f(): - return rposix.stat(self.path).st_mtime + return rposix_stat.stat(self.path).st_mtime if sys.platform == 'win32': # double vs. float, be satisfied with sub-millisec resolution assert abs(interpret(f, []) - os.stat(self.ufilename).st_mtime) < 1e-4 diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py --- a/rpython/rtyper/module/ll_os.py +++ b/rpython/rtyper/module/ll_os.py @@ -119,40 +119,6 @@ ) -class RegisterOs(BaseLazyRegistering): - - def __init__(self): - self.configure(CConfig) - -# --------------------------- os.stat & variants --------------------------- - - @registering(os.fstat) - def register_os_fstat(self): - from rpython.rtyper.module import ll_os_stat - return ll_os_stat.register_stat_variant('fstat', StringTraits()) - - @registering_str_unicode(os.stat) - def register_os_stat(self, traits): - from rpython.rtyper.module import ll_os_stat - return ll_os_stat.register_stat_variant('stat', traits) - - @registering_str_unicode(os.lstat) - def register_os_lstat(self, traits): - from rpython.rtyper.module import ll_os_stat - return ll_os_stat.register_stat_variant('lstat', traits) - - @registering_if(os, 'fstatvfs') - def register_os_fstatvfs(self): - from rpython.rtyper.module import ll_os_stat - return ll_os_stat.register_statvfs_variant('fstatvfs', StringTraits()) - - if hasattr(os, 'statvfs'): - @registering_str_unicode(os.statvfs) - def register_os_statvfs(self, traits): - from rpython.rtyper.module import ll_os_stat - return ll_os_stat.register_statvfs_variant('statvfs', traits) - - # ____________________________________________________________ # Support for os.environ diff --git a/rpython/rtyper/module/ll_os_stat.py b/rpython/rtyper/module/ll_os_stat.py deleted file mode 100644 --- a/rpython/rtyper/module/ll_os_stat.py +++ /dev/null @@ -1,592 +0,0 @@ -"""Annotation and rtyping support for the result of os.stat(), os.lstat() -and os.fstat(). In RPython like in plain Python the stat result can be -indexed like a tuple but also exposes the st_xxx attributes. -""" - -import os -import sys - -from rpython.annotator import model as annmodel -from rpython.rtyper.llannotation import lltype_to_annotation -from rpython.rlib import rposix -from rpython.rlib.rarithmetic import intmask -from rpython.rtyper import extregistry -from rpython.rtyper.annlowlevel import hlstr -from rpython.rtyper.extfunc import extdef -from rpython.rtyper.lltypesystem import rffi, lltype -from rpython.rtyper.rtuple import TUPLE_TYPE -from rpython.rtyper.tool import rffi_platform as platform -from rpython.tool.pairtype import pairtype -from rpython.tool.sourcetools import func_renamer -from rpython.translator.tool.cbuild import ExternalCompilationInfo - -# Support for float times is here. -# - ALL_STAT_FIELDS contains Float fields if the system can retrieve -# sub-second timestamps. -# - TIMESPEC is defined when the "struct stat" contains st_atim field. - -if sys.platform.startswith('linux') or sys.platform.startswith('openbsd'): - TIMESPEC = platform.Struct('struct timespec', - [('tv_sec', rffi.TIME_T), - ('tv_nsec', rffi.LONG)]) -else: - TIMESPEC = None - -# all possible fields - some of them are not available on all platforms -ALL_STAT_FIELDS = [ - ("st_mode", lltype.Signed), - ("st_ino", lltype.SignedLongLong), - ("st_dev", lltype.SignedLongLong), - ("st_nlink", lltype.Signed), - ("st_uid", lltype.Signed), - ("st_gid", lltype.Signed), - ("st_size", lltype.SignedLongLong), - ("st_atime", lltype.Float), - ("st_mtime", lltype.Float), - ("st_ctime", lltype.Float), - ("st_blksize", lltype.Signed), - ("st_blocks", lltype.Signed), - ("st_rdev", lltype.Signed), - ("st_flags", lltype.Signed), - #("st_gen", lltype.Signed), -- new in CPy 2.5, not implemented - #("st_birthtime", lltype.Float), -- new in CPy 2.5, not implemented -] -N_INDEXABLE_FIELDS = 10 - -# For OO backends, expose only the portable fields (the first 10). -PORTABLE_STAT_FIELDS = ALL_STAT_FIELDS[:N_INDEXABLE_FIELDS] - -STATVFS_FIELDS = [ - ("f_bsize", lltype.Signed), - ("f_frsize", lltype.Signed), - ("f_blocks", lltype.Signed), - ("f_bfree", lltype.Signed), - ("f_bavail", lltype.Signed), - ("f_files", lltype.Signed), - ("f_ffree", lltype.Signed), - ("f_favail", lltype.Signed), - ("f_flag", lltype.Signed), - ("f_namemax", lltype.Signed), -] - - -# ____________________________________________________________ -# -# Annotation support - -class SomeStatResult(annmodel.SomeObject): - knowntype = os.stat_result - - def rtyper_makerepr(self, rtyper): - from rpython.rtyper.module import r_os_stat - return r_os_stat.StatResultRepr(rtyper) - - def rtyper_makekey(self): - return self.__class__, - - def getattr(self, s_attr): - assert s_attr.is_constant(), "non-constant attr name in getattr()" - attrname = s_attr.const - TYPE = STAT_FIELD_TYPES[attrname] - return lltype_to_annotation(TYPE) - - def _get_rmarshall_support_(self): # for rlib.rmarshal - # reduce and recreate stat_result objects from 10-tuples - # (we ignore the extra values here for simplicity and portability) - def stat_result_reduce(st): - return (st[0], st[1], st[2], st[3], st[4], - st[5], st[6], st[7], st[8], st[9]) - - def stat_result_recreate(tup): - return make_stat_result(tup + extra_zeroes) - s_reduced = annmodel.SomeTuple([lltype_to_annotation(TYPE) - for name, TYPE in PORTABLE_STAT_FIELDS]) - extra_zeroes = (0,) * (len(STAT_FIELDS) - len(PORTABLE_STAT_FIELDS)) - return s_reduced, stat_result_reduce, stat_result_recreate - - -class SomeStatvfsResult(annmodel.SomeObject): - if hasattr(os, 'statvfs_result'): - knowntype = os.statvfs_result - else: - knowntype = None # will not be used - - def rtyper_makerepr(self, rtyper): - from rpython.rtyper.module import r_os_stat - return r_os_stat.StatvfsResultRepr(rtyper) - - def rtyper_makekey(self): - return self.__class__, - - def getattr(self, s_attr): - assert s_attr.is_constant() - TYPE = STATVFS_FIELD_TYPES[s_attr.const] - return lltype_to_annotation(TYPE) - - -class __extend__(pairtype(SomeStatResult, annmodel.SomeInteger)): - def getitem((s_sta, s_int)): - assert s_int.is_constant(), "os.stat()[index]: index must be constant" - index = s_int.const - assert 0 <= index < N_INDEXABLE_FIELDS, "os.stat()[index] out of range" - name, TYPE = STAT_FIELDS[index] - return lltype_to_annotation(TYPE) - - -class __extend__(pairtype(SomeStatvfsResult, annmodel.SomeInteger)): - def getitem((s_stat, s_int)): - assert s_int.is_constant() - name, TYPE = STATVFS_FIELDS[s_int.const] - return lltype_to_annotation(TYPE) - - -s_StatResult = SomeStatResult() -s_StatvfsResult = SomeStatvfsResult() - - -def make_stat_result(tup): - """Turn a tuple into an os.stat_result object.""" - positional = tup[:N_INDEXABLE_FIELDS] - kwds = {} - for i, name in enumerate(STAT_FIELD_NAMES[N_INDEXABLE_FIELDS:]): - kwds[name] = tup[N_INDEXABLE_FIELDS + i] - return os.stat_result(positional, kwds) - - -def make_statvfs_result(tup): - return os.statvfs_result(tup) - - -class MakeStatResultEntry(extregistry.ExtRegistryEntry): - _about_ = make_stat_result - - def compute_result_annotation(self, s_tup): - return s_StatResult - - def specialize_call(self, hop): - from rpython.rtyper.module import r_os_stat - return r_os_stat.specialize_make_stat_result(hop) - - -class MakeStatvfsResultEntry(extregistry.ExtRegistryEntry): - _about_ = make_statvfs_result - - def compute_result_annotation(self, s_tup): - return s_StatvfsResult - - def specialize_call(self, hop): - from rpython.rtyper.module import r_os_stat - return r_os_stat.specialize_make_statvfs_result(hop) - -# ____________________________________________________________ -# -# RFFI support - -if sys.platform.startswith('win'): - _name_struct_stat = '_stati64' - INCLUDES = ['sys/types.h', 'sys/stat.h', 'sys/statvfs.h'] -else: - if sys.platform.startswith('linux'): - _name_struct_stat = 'stat64' - else: - _name_struct_stat = 'stat' - INCLUDES = ['sys/types.h', 'sys/stat.h', 'sys/statvfs.h', 'unistd.h'] - -compilation_info = ExternalCompilationInfo( - # This must be set to 64 on some systems to enable large file support. - #pre_include_bits = ['#define _FILE_OFFSET_BITS 64'], - # ^^^ nowadays it's always set in all C files we produce. - includes=INCLUDES -) - -if TIMESPEC is not None: - class CConfig_for_timespec: - _compilation_info_ = compilation_info - TIMESPEC = TIMESPEC - TIMESPEC = lltype.Ptr( - platform.configure(CConfig_for_timespec)['TIMESPEC']) - - -def posix_declaration(try_to_add=None): - global STAT_STRUCT, STATVFS_STRUCT - - LL_STAT_FIELDS = STAT_FIELDS[:] - if try_to_add: - LL_STAT_FIELDS.append(try_to_add) - - if TIMESPEC is not None: - - def _expand(lst, originalname, timespecname): - for i, (_name, _TYPE) in enumerate(lst): - if _name == originalname: - # replace the 'st_atime' field of type rffi.DOUBLE - # with a field 'st_atim' of type 'struct timespec' - lst[i] = (timespecname, TIMESPEC.TO) - break - - _expand(LL_STAT_FIELDS, 'st_atime', 'st_atim') - _expand(LL_STAT_FIELDS, 'st_mtime', 'st_mtim') - _expand(LL_STAT_FIELDS, 'st_ctime', 'st_ctim') - - del _expand - else: - # Replace float fields with integers - for name in ('st_atime', 'st_mtime', 'st_ctime', 'st_birthtime'): - for i, (_name, _TYPE) in enumerate(LL_STAT_FIELDS): - if _name == name: - LL_STAT_FIELDS[i] = (_name, lltype.Signed) - break - - class CConfig: - _compilation_info_ = compilation_info - STAT_STRUCT = platform.Struct('struct %s' % _name_struct_stat, LL_STAT_FIELDS) - STATVFS_STRUCT = platform.Struct('struct statvfs', STATVFS_FIELDS) - - try: - config = platform.configure(CConfig, ignore_errors=try_to_add is not None) - except platform.CompilationError: - if try_to_add: - return # failed to add this field, give up - raise - - STAT_STRUCT = lltype.Ptr(config['STAT_STRUCT']) - STATVFS_STRUCT = lltype.Ptr(config['STATVFS_STRUCT']) - if try_to_add: - STAT_FIELDS.append(try_to_add) - - -# This lists only the fields that have been found on the underlying platform. -# Initially only the PORTABLE_STAT_FIELDS, but more may be added by the -# following loop. -STAT_FIELDS = PORTABLE_STAT_FIELDS[:] - -if sys.platform != 'win32': - posix_declaration() - for _i in range(len(PORTABLE_STAT_FIELDS), len(ALL_STAT_FIELDS)): - posix_declaration(ALL_STAT_FIELDS[_i]) - del _i - -# these two global vars only list the fields defined in the underlying platform -STAT_FIELD_TYPES = dict(STAT_FIELDS) # {'st_xxx': TYPE} -STAT_FIELD_NAMES = [_name for (_name, _TYPE) in STAT_FIELDS] -del _name, _TYPE - -STATVFS_FIELD_TYPES = dict(STATVFS_FIELDS) -STATVFS_FIELD_NAMES = [name for name, tp in STATVFS_FIELDS] - - -def build_stat_result(st): - # only for LL backends - if TIMESPEC is not None: - atim = st.c_st_atim; atime = int(atim.c_tv_sec) + 1E-9 * int(atim.c_tv_nsec) - mtim = st.c_st_mtim; mtime = int(mtim.c_tv_sec) + 1E-9 * int(mtim.c_tv_nsec) - ctim = st.c_st_ctim; ctime = int(ctim.c_tv_sec) + 1E-9 * int(ctim.c_tv_nsec) - else: - atime = st.c_st_atime - mtime = st.c_st_mtime - ctime = st.c_st_ctime - - result = (st.c_st_mode, - st.c_st_ino, - st.c_st_dev, - st.c_st_nlink, - st.c_st_uid, - st.c_st_gid, - st.c_st_size, - atime, - mtime, - ctime) - - if "st_blksize" in STAT_FIELD_TYPES: result += (st.c_st_blksize,) - if "st_blocks" in STAT_FIELD_TYPES: result += (st.c_st_blocks,) - if "st_rdev" in STAT_FIELD_TYPES: result += (st.c_st_rdev,) - if "st_flags" in STAT_FIELD_TYPES: result += (st.c_st_flags,) - - return make_stat_result(result) - - -def build_statvfs_result(st): - return make_statvfs_result(( - st.c_f_bsize, - st.c_f_frsize, - st.c_f_blocks, - st.c_f_bfree, - st.c_f_bavail, - st.c_f_files, - st.c_f_ffree, - st.c_f_favail, - st.c_f_flag, - st.c_f_namemax - )) - - -def register_stat_variant(name, traits): - if name != 'fstat': - arg_is_path = True - s_arg = traits.str0 - ARG1 = traits.CCHARP - else: - arg_is_path = False - s_arg = int - ARG1 = rffi.INT - - if sys.platform == 'win32': - # See Win32 implementation below - posix_stat_llimpl = make_win32_stat_impl(name, traits) - - return extdef( - [s_arg], s_StatResult, traits.ll_os_name(name), - llimpl=posix_stat_llimpl) - - if sys.platform.startswith('linux'): - # because we always use _FILE_OFFSET_BITS 64 - this helps things work that are not a c compiler - _functions = {'stat': 'stat64', - 'fstat': 'fstat64', - 'lstat': 'lstat64'} - c_func_name = _functions[name] - else: - c_func_name = name - - posix_mystat = rffi.llexternal(c_func_name, - [ARG1, STAT_STRUCT], rffi.INT, - compilation_info=compilation_info, - save_err=rffi.RFFI_SAVE_ERRNO) - - @func_renamer('os_%s_llimpl' % (name,)) - def posix_stat_llimpl(arg): - stresult = lltype.malloc(STAT_STRUCT.TO, flavor='raw') - try: - if arg_is_path: - arg = traits.str2charp(arg) - error = rffi.cast(rffi.LONG, posix_mystat(arg, stresult)) - if arg_is_path: - traits.free_charp(arg) - if error != 0: - raise OSError(rposix.get_saved_errno(), "os_?stat failed") - return build_stat_result(stresult) - finally: - lltype.free(stresult, flavor='raw') - - @func_renamer('os_%s_fake' % (name,)) - def posix_fakeimpl(arg): - if s_arg == traits.str0: - arg = hlstr(arg) - st = getattr(os, name)(arg) - fields = [TYPE for fieldname, TYPE in STAT_FIELDS] - TP = TUPLE_TYPE(fields) - ll_tup = lltype.malloc(TP.TO) - for i, (fieldname, TYPE) in enumerate(STAT_FIELDS): - val = getattr(st, fieldname) - if isinstance(TYPE, lltype.Number): - rffi.setintfield(ll_tup, 'item%d' % i, int(val)) - elif TYPE is lltype.Float: - setattr(ll_tup, 'item%d' % i, float(val)) - else: - setattr(ll_tup, 'item%d' % i, val) - return ll_tup - - return extdef( - [s_arg], s_StatResult, "ll_os.ll_os_%s" % (name,), - llimpl=posix_stat_llimpl, llfakeimpl=posix_fakeimpl) - - -def register_statvfs_variant(name, traits): - if name != 'fstatvfs': - arg_is_path = True - s_arg = traits.str0 - ARG1 = traits.CCHARP - else: - arg_is_path = False - s_arg = int - ARG1 = rffi.INT - - posix_mystatvfs = rffi.llexternal(name, - [ARG1, STATVFS_STRUCT], rffi.INT, - compilation_info=compilation_info, - save_err=rffi.RFFI_SAVE_ERRNO) - - @func_renamer('os_%s_llimpl' % (name,)) - def posix_statvfs_llimpl(arg): - stresult = lltype.malloc(STATVFS_STRUCT.TO, flavor='raw') - try: - if arg_is_path: - arg = traits.str2charp(arg) - error = rffi.cast(rffi.LONG, posix_mystatvfs(arg, stresult)) - if arg_is_path: - traits.free_charp(arg) - if error != 0: - raise OSError(rposix.get_saved_errno(), "os_?statvfs failed") - return build_statvfs_result(stresult) - finally: - lltype.free(stresult, flavor='raw') - - @func_renamer('os_%s_fake' % (name,)) - def posix_fakeimpl(arg): - if s_arg == traits.str0: - arg = hlstr(arg) - st = getattr(os, name)(arg) - fields = [TYPE for fieldname, TYPE in STATVFS_FIELDS] - TP = TUPLE_TYPE(fields) - ll_tup = lltype.malloc(TP.TO) - for i, (fieldname, TYPE) in enumerate(STATVFS_FIELDS): - val = getattr(st, fieldname) - rffi.setintfield(ll_tup, 'item%d' % i, int(val)) - return ll_tup - - return extdef( - [s_arg], s_StatvfsResult, "ll_os.ll_os_%s" % (name,), - llimpl=posix_statvfs_llimpl, llfakeimpl=posix_fakeimpl - ) - - -def make_win32_stat_impl(name, traits): - from rpython.rlib import rwin32 - from rpython.rlib.rwin32file import make_win32_traits - win32traits = make_win32_traits(traits) - - # The CRT of Windows has a number of flaws wrt. its stat() implementation: - # - time stamps are restricted to second resolution - # - file modification times suffer from forth-and-back conversions between - # UTC and local time - # Therefore, we implement our own stat, based on the Win32 API directly. - from rpython.rtyper.tool import rffi_platform as platform - from rpython.translator.tool.cbuild import ExternalCompilationInfo - from rpython.rlib import rwin32 - - assert len(STAT_FIELDS) == 10 # no extra fields on Windows - - def attributes_to_mode(attributes): - m = 0 - attributes = intmask(attributes) - if attributes & win32traits.FILE_ATTRIBUTE_DIRECTORY: - m |= win32traits._S_IFDIR | 0111 # IFEXEC for user,group,other - else: - m |= win32traits._S_IFREG - if attributes & win32traits.FILE_ATTRIBUTE_READONLY: - m |= 0444 - else: - m |= 0666 - return m - - def attribute_data_to_stat(info): - st_mode = attributes_to_mode(info.c_dwFileAttributes) - st_size = make_longlong(info.c_nFileSizeHigh, info.c_nFileSizeLow) - ctime = FILE_TIME_to_time_t_float(info.c_ftCreationTime) - mtime = FILE_TIME_to_time_t_float(info.c_ftLastWriteTime) - atime = FILE_TIME_to_time_t_float(info.c_ftLastAccessTime) - - result = (st_mode, - 0, 0, 0, 0, 0, - st_size, - atime, mtime, ctime) - - return make_stat_result(result) - - def by_handle_info_to_stat(info): - # similar to the one above - st_mode = attributes_to_mode(info.c_dwFileAttributes) - st_size = make_longlong(info.c_nFileSizeHigh, info.c_nFileSizeLow) - ctime = FILE_TIME_to_time_t_float(info.c_ftCreationTime) - mtime = FILE_TIME_to_time_t_float(info.c_ftLastWriteTime) - atime = FILE_TIME_to_time_t_float(info.c_ftLastAccessTime) - - # specific to fstat() - st_ino = make_longlong(info.c_nFileIndexHigh, info.c_nFileIndexLow) - st_nlink = info.c_nNumberOfLinks - - result = (st_mode, - st_ino, 0, st_nlink, 0, 0, - st_size, - atime, mtime, ctime) - - return make_stat_result(result) - - def attributes_from_dir(l_path, data): - filedata = lltype.malloc(win32traits.WIN32_FIND_DATA, flavor='raw') - try: - hFindFile = win32traits.FindFirstFile(l_path, filedata) - if hFindFile == rwin32.INVALID_HANDLE_VALUE: - return 0 - win32traits.FindClose(hFindFile) - data.c_dwFileAttributes = filedata.c_dwFileAttributes - rffi.structcopy(data.c_ftCreationTime, filedata.c_ftCreationTime) - rffi.structcopy(data.c_ftLastAccessTime, filedata.c_ftLastAccessTime) - rffi.structcopy(data.c_ftLastWriteTime, filedata.c_ftLastWriteTime) - data.c_nFileSizeHigh = filedata.c_nFileSizeHigh - data.c_nFileSizeLow = filedata.c_nFileSizeLow - return 1 - finally: - lltype.free(filedata, flavor='raw') - - def win32_stat_llimpl(path): - data = lltype.malloc(win32traits.WIN32_FILE_ATTRIBUTE_DATA, flavor='raw') - try: - l_path = traits.str2charp(path) - res = win32traits.GetFileAttributesEx(l_path, win32traits.GetFileExInfoStandard, data) - errcode = rwin32.GetLastError_saved() - if res == 0: - if errcode == win32traits.ERROR_SHARING_VIOLATION: - res = attributes_from_dir(l_path, data) - errcode = rwin32.GetLastError_saved() - traits.free_charp(l_path) - if res == 0: - raise WindowsError(errcode, "os_stat failed") - return attribute_data_to_stat(data) - finally: - lltype.free(data, flavor='raw') - - def win32_fstat_llimpl(fd): - handle = rwin32.get_osfhandle(fd) - filetype = win32traits.GetFileType(handle) - if filetype == win32traits.FILE_TYPE_CHAR: - # console or LPT device - return make_stat_result((win32traits._S_IFCHR, - 0, 0, 0, 0, 0, - 0, 0, 0, 0)) - elif filetype == win32traits.FILE_TYPE_PIPE: - # socket or named pipe - return make_stat_result((win32traits._S_IFIFO, - 0, 0, 0, 0, 0, - 0, 0, 0, 0)) - elif filetype == win32traits.FILE_TYPE_UNKNOWN: - error = rwin32.GetLastError_saved() - if error != 0: - raise WindowsError(error, "os_fstat failed") - # else: unknown but valid file - - # normal disk file (FILE_TYPE_DISK) - info = lltype.malloc(win32traits.BY_HANDLE_FILE_INFORMATION, - flavor='raw', zero=True) - try: - res = win32traits.GetFileInformationByHandle(handle, info) - if res == 0: - raise WindowsError(rwin32.GetLastError_saved(), - "os_fstat failed") - return by_handle_info_to_stat(info) - finally: - lltype.free(info, flavor='raw') - - if name == 'fstat': - return win32_fstat_llimpl - else: - return win32_stat_llimpl - - -#__________________________________________________ -# Helper functions for win32 - -def make_longlong(high, low): - return (rffi.r_longlong(high) << 32) + rffi.r_longlong(low) - -# Seconds between 1.1.1601 and 1.1.1970 -secs_between_epochs = rffi.r_longlong(11644473600) - -def FILE_TIME_to_time_t_float(filetime): - ft = make_longlong(filetime.c_dwHighDateTime, filetime.c_dwLowDateTime) - # FILETIME is in units of 100 nsec - return float(ft) * (1.0 / 10000000.0) - secs_between_epochs - -def time_t_to_FILE_TIME(time, filetime): - ft = rffi.r_longlong((time + secs_between_epochs) * 10000000) - filetime.c_dwHighDateTime = rffi.r_uint(ft >> 32) - filetime.c_dwLowDateTime = rffi.r_uint(ft) # masking off high bits diff --git a/rpython/rtyper/module/r_os_stat.py b/rpython/rtyper/module/r_os_stat.py deleted file mode 100644 --- a/rpython/rtyper/module/r_os_stat.py +++ /dev/null @@ -1,122 +0,0 @@ -""" -RTyping support for os.stat_result objects. -They are rtyped just like a tuple of the correct length supporting -only indexing and the st_xxx attributes. We need a custom StatResultRepr -because when rtyping for LL backends we have extra platform-dependent -items at the end of the tuple, but for OO backends we only want the -portable items. This allows the OO backends to assume a fixed shape for -the tuples returned by os.stat(). -""" -from rpython.annotator import model as annmodel -from rpython.rtyper.llannotation import lltype_to_annotation -from rpython.flowspace.model import Constant -from rpython.flowspace.operation import op -from rpython.tool.pairtype import pairtype -from rpython.rtyper.rmodel import Repr -from rpython.rtyper.rint import IntegerRepr -from rpython.rtyper.error import TyperError -from rpython.rtyper.module import ll_os_stat - - -class StatResultRepr(Repr): - - def __init__(self, rtyper): - self.rtyper = rtyper - self.stat_fields = ll_os_stat.STAT_FIELDS - - self.stat_field_indexes = {} - for i, (name, TYPE) in enumerate(self.stat_fields): - self.stat_field_indexes[name] = i - - self.s_tuple = annmodel.SomeTuple([lltype_to_annotation(TYPE) - for name, TYPE in self.stat_fields]) - self.r_tuple = rtyper.getrepr(self.s_tuple) - self.lowleveltype = self.r_tuple.lowleveltype - - def redispatch_getfield(self, hop, index): - rtyper = self.rtyper - s_index = rtyper.annotator.bookkeeper.immutablevalue(index) - hop2 = hop.copy() - spaceop = op.getitem(hop.args_v[0], Constant(index)) - spaceop.result = hop.spaceop.result - hop2.spaceop = spaceop - hop2.args_v = spaceop.args - hop2.args_s = [self.s_tuple, s_index] - hop2.args_r = [self.r_tuple, rtyper.getrepr(s_index)] - return hop2.dispatch() - - def rtype_getattr(self, hop): - s_attr = hop.args_s[1] - attr = s_attr.const - try: - index = self.stat_field_indexes[attr] - except KeyError: - raise TyperError("os.stat().%s: field not available" % (attr,)) - return self.redispatch_getfield(hop, index) - - -class __extend__(pairtype(StatResultRepr, IntegerRepr)): - - def rtype_getitem((r_sta, r_int), hop): - s_int = hop.args_s[1] - index = s_int.const - return r_sta.redispatch_getfield(hop, index) - - -def specialize_make_stat_result(hop): - r_StatResult = hop.rtyper.getrepr(ll_os_stat.s_StatResult) - [v_result] = hop.inputargs(r_StatResult.r_tuple) - # no-op conversion from r_StatResult.r_tuple to r_StatResult - hop.exception_cannot_occur() - return v_result - - -class StatvfsResultRepr(Repr): - - def __init__(self, rtyper): - self.rtyper = rtyper - self.statvfs_fields = ll_os_stat.STATVFS_FIELDS - - self.statvfs_field_indexes = {} - for i, (name, TYPE) in enumerate(self.statvfs_fields): - self.statvfs_field_indexes[name] = i - - self.s_tuple = annmodel.SomeTuple([lltype_to_annotation(TYPE) - for name, TYPE in self.statvfs_fields]) - self.r_tuple = rtyper.getrepr(self.s_tuple) - self.lowleveltype = self.r_tuple.lowleveltype - - def redispatch_getfield(self, hop, index): - rtyper = self.rtyper - s_index = rtyper.annotator.bookkeeper.immutablevalue(index) - hop2 = hop.copy() - spaceop = op.getitem(hop.args_v[0], Constant(index)) - spaceop.result = hop.spaceop.result - hop2.spaceop = spaceop - hop2.args_v = spaceop.args - hop2.args_s = [self.s_tuple, s_index] - hop2.args_r = [self.r_tuple, rtyper.getrepr(s_index)] - return hop2.dispatch() - - def rtype_getattr(self, hop): - s_attr = hop.args_s[1] - attr = s_attr.const - try: - index = self.statvfs_field_indexes[attr] - except KeyError: - raise TyperError("os.statvfs().%s: field not available" % (attr,)) - return self.redispatch_getfield(hop, index) - - -class __extend__(pairtype(StatvfsResultRepr, IntegerRepr)): - def rtype_getitem((r_sta, r_int), hop): - s_int = hop.args_s[1] - index = s_int.const - return r_sta.redispatch_getfield(hop, index) - - -def specialize_make_statvfs_result(hop): - r_StatvfsResult = hop.rtyper.getrepr(ll_os_stat.s_StatvfsResult) - [v_result] = hop.inputargs(r_StatvfsResult.r_tuple) - hop.exception_cannot_occur() - return v_result From noreply at buildbot.pypy.org Mon Apr 6 13:58:32 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 6 Apr 2015 13:58:32 +0200 (CEST) Subject: [pypy-commit] pypy more-rposix: Fix translation Message-ID: <20150406115832.72F381C0204@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: more-rposix Changeset: r76722:aaf75c2912cb Date: 2015-04-06 13:58 +0200 http://bitbucket.org/pypy/pypy/changeset/aaf75c2912cb/ Log: Fix translation diff --git a/rpython/rlib/rposix_stat.py b/rpython/rlib/rposix_stat.py --- a/rpython/rlib/rposix_stat.py +++ b/rpython/rlib/rposix_stat.py @@ -417,6 +417,7 @@ return build_statvfs_result(stresult) @replace_os_function('statvfs') + at specialize.argtype(0) def statvfs(path): with lltype.scoped_alloc(STATVFS_STRUCT.TO) as stresult: arg = _as_bytes0(path) From noreply at buildbot.pypy.org Mon Apr 6 15:32:16 2015 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 6 Apr 2015 15:32:16 +0200 (CEST) Subject: [pypy-commit] pypy vmprof: strip the topmost part of the code, with rationale Message-ID: <20150406133216.4E9BB1C0EE0@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76723:9efba357dd88 Date: 2015-04-06 15:22 +0200 http://bitbucket.org/pypy/pypy/changeset/9efba357dd88/ Log: strip the topmost part of the code, with rationale diff --git a/pypy/module/_vmprof/src/get_custom_offset.c b/pypy/module/_vmprof/src/get_custom_offset.c --- a/pypy/module/_vmprof/src/get_custom_offset.c +++ b/pypy/module/_vmprof/src/get_custom_offset.c @@ -48,6 +48,12 @@ break; result[n++] = (void *)id; } + // we strip the topmost part - the reason is that it's either + // represented in the jitted caller or it's not jitted (we have the + // same function essentially twice + if (n != max_depth) { + n--; + } k = 0; while (k < (n - start) / 2) { tmp = result[start + k]; From noreply at buildbot.pypy.org Mon Apr 6 15:32:17 2015 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 6 Apr 2015 15:32:17 +0200 (CEST) Subject: [pypy-commit] pypy vmprof: ups, micro = 1e-6 Message-ID: <20150406133217.6CACE1C0EE0@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76724:dbd25781d4a2 Date: 2015-04-06 15:31 +0200 http://bitbucket.org/pypy/pypy/changeset/dbd25781d4a2/ Log: ups, micro = 1e-6 diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py --- a/pypy/module/_vmprof/interp_vmprof.py +++ b/pypy/module/_vmprof/interp_vmprof.py @@ -224,7 +224,7 @@ # try: period_usec = ovfcheck_float_to_int(period * 1000000.0 + 0.5) - if period_usec <= 0 or period_usec >= 1e9: + if period_usec <= 0 or period_usec >= 1e6: # we don't want seconds here at all raise ValueError except (ValueError, OverflowError): From noreply at buildbot.pypy.org Mon Apr 6 15:45:37 2015 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 6 Apr 2015 15:45:37 +0200 (CEST) Subject: [pypy-commit] pypy vmprof: more like that Message-ID: <20150406134537.588DE1C0334@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76725:6cf070128449 Date: 2015-04-06 15:45 +0200 http://bitbucket.org/pypy/pypy/changeset/6cf070128449/ Log: more like that diff --git a/pypy/module/_vmprof/src/get_custom_offset.c b/pypy/module/_vmprof/src/get_custom_offset.c --- a/pypy/module/_vmprof/src/get_custom_offset.c +++ b/pypy/module/_vmprof/src/get_custom_offset.c @@ -51,9 +51,6 @@ // we strip the topmost part - the reason is that it's either // represented in the jitted caller or it's not jitted (we have the // same function essentially twice - if (n != max_depth) { - n--; - } k = 0; while (k < (n - start) / 2) { tmp = result[start + k]; @@ -61,5 +58,8 @@ result[n - k - 1] = tmp; k++; } + if (n != max_depth) { + n--; + } return n; } From noreply at buildbot.pypy.org Mon Apr 6 17:23:21 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 6 Apr 2015 17:23:21 +0200 (CEST) Subject: [pypy-commit] pypy default: In the default case, try harder to find /usr/include/tclX.Y Message-ID: <20150406152321.3014F1C0EEE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76726:0b13b369733e Date: 2015-04-06 17:23 +0200 http://bitbucket.org/pypy/pypy/changeset/0b13b369733e/ Log: In the default case, try harder to find /usr/include/tclX.Y diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py --- a/lib_pypy/_tkinter/tklib.py +++ b/lib_pypy/_tkinter/tklib.py @@ -135,7 +135,7 @@ linklibs = ['tcl', 'tk'] libdirs = [] else: - incdirs=['/usr/include/tcl'] + incdirs=['/usr/include/tcl', '/usr/include/tcl8.6', '/usr/include/tcl8.5'] linklibs=['tcl', 'tk'] libdirs = [] From noreply at buildbot.pypy.org Mon Apr 6 18:20:56 2015 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 6 Apr 2015 18:20:56 +0200 (CEST) Subject: [pypy-commit] pypy vmprof: add a direct test Message-ID: <20150406162056.D4A331C0EEE@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76727:7db06b25b25f Date: 2015-04-06 18:20 +0200 http://bitbucket.org/pypy/pypy/changeset/7db06b25b25f/ Log: add a direct test diff --git a/pypy/module/_vmprof/src/get_custom_offset.c b/pypy/module/_vmprof/src/get_custom_offset.c --- a/pypy/module/_vmprof/src/get_custom_offset.c +++ b/pypy/module/_vmprof/src/get_custom_offset.c @@ -16,7 +16,7 @@ return !pypy_codemap_currently_invalid; } -static ptrdiff_t vmprof_unw_get_custom_offset(void* ip, unw_cursor_t *cp) { +static ptrdiff_t vmprof_unw_get_custom_offset(void* ip, void *cp) { intptr_t ip_l = (intptr_t)ip; return pypy_jit_stack_depth_at_loc(ip_l); } diff --git a/pypy/module/_vmprof/test/test_direct.py b/pypy/module/_vmprof/test/test_direct.py new file mode 100644 --- /dev/null +++ b/pypy/module/_vmprof/test/test_direct.py @@ -0,0 +1,66 @@ + +import cffi, py + +srcdir = py.path.local(__file__).join("..", "..", "src") + +ffi = cffi.FFI() +ffi.cdef(""" +long vmprof_write_header_for_jit_addr(void **, long, void*, int); +void *pypy_find_codemap_at_addr(long addr, long *start_addr); +long pypy_yield_codemap_at_addr(void *codemap_raw, long addr, + long *current_pos_addr); +long buffer[]; +""") + +lib = ffi.verify(""" +volatile int pypy_codemap_currently_invalid = 0; + +long buffer[] = {0, 0, 0, 0, 0}; + + + +void *pypy_find_codemap_at_addr(long addr, long *start_addr) +{ + return (void*)buffer; +} + +long pypy_yield_codemap_at_addr(void *codemap_raw, long addr, + long *current_pos_addr) +{ + long c = *current_pos_addr; + if (c >= 5) + return 0; + *current_pos_addr = c + 1; + return *((long*)codemap_raw + c); +} + + +""" + open(str(srcdir.join("get_custom_offset.c"))).read()) + +class TestDirect(object): + def test_infrastructure(self): + cont = ffi.new("long[1]", [0]) + buf = lib.pypy_find_codemap_at_addr(0, cont) + assert buf + cont[0] = 0 + next_addr = lib.pypy_yield_codemap_at_addr(buf, 0, cont) + assert cont[0] == 1 + assert not next_addr + lib.buffer[0] = 13 + cont[0] = 0 + next_addr = lib.pypy_yield_codemap_at_addr(buf, 0, cont) + assert int(ffi.cast("long", next_addr)) == 13 + + def test_write_header_for_jit_addr(self): + lib.buffer[0] = 4 + lib.buffer[1] = 8 + lib.buffer[2] = 12 + lib.buffer[3] = 16 + lib.buffer[4] = 0 + buf = ffi.new("long[5]", [0] * 5) + result = ffi.cast("void**", buf) + res = lib.vmprof_write_header_for_jit_addr(result, 0, ffi.NULL, 100) + assert res == 3 + assert buf[0] == 16 + assert buf[1] == 12 + assert buf[2] == 8 From noreply at buildbot.pypy.org Mon Apr 6 18:39:31 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 6 Apr 2015 18:39:31 +0200 (CEST) Subject: [pypy-commit] pypy default: Meh, see comment. Message-ID: <20150406163931.3BB101C0204@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76728:36bfec91449e Date: 2015-04-06 18:39 +0200 http://bitbucket.org/pypy/pypy/changeset/36bfec91449e/ Log: Meh, see comment. diff --git a/rpython/memory/gctransform/shadowstack.py b/rpython/memory/gctransform/shadowstack.py --- a/rpython/memory/gctransform/shadowstack.py +++ b/rpython/memory/gctransform/shadowstack.py @@ -464,6 +464,13 @@ def customtrace(gc, obj, callback, arg): obj = llmemory.cast_adr_to_ptr(obj, SHADOWSTACKREFPTR) + index = obj.fsindex + if index > 0: + # Haaaaaaack: fullstack_cache[] is just an integer, so it + # doesn't follow the SHADOWSTACKREF when it moves. But we + # know this customtrace() will be called just after the + # move. So we fix the fullstack_cache[] now... :-/ + fullstack_cache[index] = lltype.cast_ptr_to_int(obj) addr = obj.top start = obj.base while addr != start: From noreply at buildbot.pypy.org Mon Apr 6 21:05:08 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 6 Apr 2015 21:05:08 +0200 (CEST) Subject: [pypy-commit] pypy more-rposix: Some fixes for windows Message-ID: <20150406190508.538EB1C1422@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: more-rposix Changeset: r76729:ae2d76775668 Date: 2015-04-06 21:02 +0200 http://bitbucket.org/pypy/pypy/changeset/ae2d76775668/ Log: Some fixes for windows diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -18,6 +18,11 @@ _WIN32 = sys.platform.startswith('win') _CYGWIN = sys.platform == 'cygwin' UNDERSCORE_ON_WIN32 = '_' if _WIN32 else '' +_MACRO_ON_POSIX = True if not _WIN32 else None + +if _WIN32: + from rpython.rlib import rwin32 + from rpython.rlib.rwin32file import make_win32_traits class CConstantErrno(CConstant): # these accessors are used when calling get_errno() or set_errno() @@ -268,7 +273,8 @@ # For now we require off_t to be the same size as LONGLONG, which is the # interface required by callers of functions that thake an argument of type # off_t. -assert OFF_T_SIZE == rffi.sizeof(rffi.LONGLONG) +if not _WIN32: + assert OFF_T_SIZE == rffi.sizeof(rffi.LONGLONG) c_dup = external(UNDERSCORE_ON_WIN32 + 'dup', [rffi.INT], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO) @@ -369,8 +375,11 @@ # with errno. def replace_os_function(name): + func = getattr(os, name, None) + if func is None: + return lambda f: f return register_replacement_for( - getattr(os, name, None), + func, sandboxed_name='ll_os.ll_os_%s' % name) @specialize.arg(0) @@ -436,7 +445,7 @@ c_lseek = external('_lseeki64' if _WIN32 else 'lseek', [rffi.INT, rffi.LONGLONG, rffi.INT], rffi.LONGLONG, - macro=True, save_err=rffi.RFFI_SAVE_ERRNO) + macro=_MACRO_ON_POSIX, save_err=rffi.RFFI_SAVE_ERRNO) @replace_os_function('lseek') def lseek(fd, pos, how): @@ -451,7 +460,7 @@ return handle_posix_error('lseek', c_lseek(fd, pos, how)) c_ftruncate = external('ftruncate', [rffi.INT, rffi.LONGLONG], rffi.INT, - macro=True, save_err=rffi.RFFI_SAVE_ERRNO) + macro=_MACRO_ON_POSIX, save_err=rffi.RFFI_SAVE_ERRNO) c_fsync = external('fsync' if not _WIN32 else '_commit', [rffi.INT], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO) c_fdatasync = external('fdatasync', [rffi.INT], rffi.INT, @@ -490,9 +499,8 @@ handle_posix_error('chdir', c_chdir(_as_bytes0(path))) else: traits = _preferred_traits(path) - from rpython.rlib.rwin32file import make_win32_traits win32traits = make_win32_traits(traits) - path = traits._as_str0(path) + path = traits.as_str0(path) # This is a reimplementation of the C library's chdir # function, but one that produces Win32 errors instead of DOS @@ -556,7 +564,6 @@ def getfullpathname(path): length = rwin32.MAX_PATH + 1 traits = _preferred_traits(path) - from rpython.rlib.rwin32file import make_win32_traits win32traits = make_win32_traits(traits) with traits.scoped_alloc_buffer(length) as buf: res = win32traits.GetFullPathName( @@ -654,7 +661,6 @@ raise OSError(error, "readdir failed") return result else: # _WIN32 case - from rpython.rlib.rwin32file import make_win32_traits traits = _preferred_traits(path) win32traits = make_win32_traits(traits) path = traits.as_str0(path) @@ -846,7 +852,7 @@ def _make_waitmacro(name): c_func = external(name, [lltype.Signed], lltype.Signed, - macro=True) + macro=_MACRO_ON_POSIX) returning_int = name in ('WEXITSTATUS', 'WSTOPSIG', 'WTERMSIG') @replace_os_function(name) @@ -1040,7 +1046,7 @@ save_err=rffi.RFFI_SAVE_ERRNO) c_mknod = external('mknod', [rffi.CCHARP, rffi.MODE_T, rffi.INT], rffi.INT, # # xxx: actually ^^^ dev_t - macro=True, save_err=rffi.RFFI_SAVE_ERRNO) + macro=_MACRO_ON_POSIX, save_err=rffi.RFFI_SAVE_ERRNO) @replace_os_function('mkfifo') @specialize.argtype(0) @@ -1212,7 +1218,7 @@ lltype.free(l_utimbuf, flavor='raw') handle_posix_error('utime', error) else: # _WIN32 case - from rpython.rlib.rwin32file import make_win32_traits, time_t_to_FILE_TIME + from rpython.rlib.rwin32file import time_t_to_FILE_TIME traits = _preferred_traits(path) win32traits = make_win32_traits(traits) path = traits.as_str0(path) diff --git a/rpython/rlib/rposix_stat.py b/rpython/rlib/rposix_stat.py --- a/rpython/rlib/rposix_stat.py +++ b/rpython/rlib/rposix_stat.py @@ -11,8 +11,7 @@ from rpython.rtyper.tool import rffi_platform as platform from rpython.rtyper.llannotation import lltype_to_annotation -from rpython.rlib.objectmodel import ( - specialize) +from rpython.rlib.objectmodel import specialize from rpython.rtyper.lltypesystem import lltype, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.rlib.rposix import ( @@ -427,6 +426,7 @@ #__________________________________________________ # Helper functions for win32 if _WIN32: + from rpython.rlib.rwin32file import FILE_TIME_to_time_t_float def make_longlong(high, low): return (rffi.r_longlong(high) << 32) + rffi.r_longlong(low) @@ -434,16 +434,6 @@ # Seconds between 1.1.1601 and 1.1.1970 secs_between_epochs = rffi.r_longlong(11644473600) - def FILE_TIME_to_time_t_float(filetime): - ft = make_longlong(filetime.c_dwHighDateTime, filetime.c_dwLowDateTime) - # FILETIME is in units of 100 nsec - return float(ft) * (1.0 / 10000000.0) - secs_between_epochs - - def time_t_to_FILE_TIME(time, filetime): - ft = rffi.r_longlong((time + secs_between_epochs) * 10000000) - filetime.c_dwHighDateTime = rffi.r_uint(ft >> 32) - filetime.c_dwLowDateTime = rffi.r_uint(ft) # masking off high bits - def win32_xstat(traits, path, traverse=False): win32traits = make_win32_traits(traits) with lltype.scoped_alloc( From noreply at buildbot.pypy.org Mon Apr 6 22:21:29 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 6 Apr 2015 22:21:29 +0200 (CEST) Subject: [pypy-commit] pypy more-rposix: Fix translation on Windows Message-ID: <20150406202129.5C13E1C1331@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: more-rposix Changeset: r76730:7cc288b7e843 Date: 2015-04-06 22:17 +0200 http://bitbucket.org/pypy/pypy/changeset/7cc288b7e843/ Log: Fix translation on Windows diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -1033,7 +1033,7 @@ handle_posix_error('rename', c_rename(_as_bytes0(path1), _as_bytes0(path2))) else: - traits = _preferred_traits(path) + traits = _preferred_traits(path1) win32traits = make_win32_traits(traits) path1 = traits.as_str0(path1) path2 = traits.as_str0(path2) @@ -1087,8 +1087,8 @@ finally: lltype.free(pwrite, flavor='raw') lltype.free(pread, flavor='raw') - fdread = _open_osfhandle(hread, 0) - fdwrite = _open_osfhandle(hwrite, 1) + fdread = c_open_osfhandle(hread, 0) + fdwrite = c_open_osfhandle(hwrite, 1) return (fdread, fdwrite) else: filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw') diff --git a/rpython/rlib/rposix_stat.py b/rpython/rlib/rposix_stat.py --- a/rpython/rlib/rposix_stat.py +++ b/rpython/rlib/rposix_stat.py @@ -14,12 +14,18 @@ from rpython.rlib.objectmodel import specialize from rpython.rtyper.lltypesystem import lltype, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo +from rpython.rlib.rarithmetic import intmask from rpython.rlib.rposix import ( - replace_os_function, handle_posix_error, _as_bytes0) + replace_os_function, handle_posix_error, _as_bytes0, + _preferred_traits, string_traits) _WIN32 = sys.platform.startswith('win') _LINUX = sys.platform.startswith('linux') +if _WIN32: + from rpython.rlib import rwin32 + from rpython.rlib.rwin32file import make_win32_traits + # Support for float times is here. # - ALL_STAT_FIELDS contains Float fields if the system can retrieve # sub-second timestamps. @@ -321,93 +327,94 @@ # Implement and register os.stat() & variants -c_fstat = rffi.llexternal('fstat64' if _LINUX else 'fstat', - [rffi.INT, STAT_STRUCT], rffi.INT, - compilation_info=compilation_info, - save_err=rffi.RFFI_SAVE_ERRNO, - macro=True) -c_stat = rffi.llexternal('stat64' if _LINUX else 'stat', - [rffi.CCHARP, STAT_STRUCT], rffi.INT, - compilation_info=compilation_info, - save_err=rffi.RFFI_SAVE_ERRNO, - macro=True) -c_lstat = rffi.llexternal('lstat64' if _LINUX else 'lstat', - [rffi.CCHARP, STAT_STRUCT], rffi.INT, - compilation_info=compilation_info, - save_err=rffi.RFFI_SAVE_ERRNO, - macro=True) +if not _WIN32: + c_fstat = rffi.llexternal('fstat64' if _LINUX else 'fstat', + [rffi.INT, STAT_STRUCT], rffi.INT, + compilation_info=compilation_info, + save_err=rffi.RFFI_SAVE_ERRNO, + macro=True) + c_stat = rffi.llexternal('stat64' if _LINUX else 'stat', + [rffi.CCHARP, STAT_STRUCT], rffi.INT, + compilation_info=compilation_info, + save_err=rffi.RFFI_SAVE_ERRNO, + macro=True) + c_lstat = rffi.llexternal('lstat64' if _LINUX else 'lstat', + [rffi.CCHARP, STAT_STRUCT], rffi.INT, + compilation_info=compilation_info, + save_err=rffi.RFFI_SAVE_ERRNO, + macro=True) -c_fstatvfs = rffi.llexternal('fstatvfs', - [rffi.INT, STATVFS_STRUCT], rffi.INT, - compilation_info=compilation_info, - save_err=rffi.RFFI_SAVE_ERRNO) -c_statvfs = rffi.llexternal('statvfs', - [rffi.CCHARP, STATVFS_STRUCT], rffi.INT, - compilation_info=compilation_info, - save_err=rffi.RFFI_SAVE_ERRNO) + c_fstatvfs = rffi.llexternal('fstatvfs', + [rffi.INT, STATVFS_STRUCT], rffi.INT, + compilation_info=compilation_info, + save_err=rffi.RFFI_SAVE_ERRNO) + c_statvfs = rffi.llexternal('statvfs', + [rffi.CCHARP, STATVFS_STRUCT], rffi.INT, + compilation_info=compilation_info, + save_err=rffi.RFFI_SAVE_ERRNO) @replace_os_function('fstat') def fstat(fd): - with lltype.scoped_alloc(STAT_STRUCT.TO) as stresult: - if not _WIN32: + if not _WIN32: + with lltype.scoped_alloc(STAT_STRUCT.TO) as stresult: handle_posix_error('fstat', c_fstat(fd, stresult)) return build_stat_result(stresult) - else: - handle = rwin32.get_osfhandle(fd) - filetype = win32traits.GetFileType(handle) - if filetype == win32traits.FILE_TYPE_CHAR: - # console or LPT device - return make_stat_result((win32traits._S_IFCHR, - 0, 0, 0, 0, 0, - 0, 0, 0, 0)) - elif filetype == win32traits.FILE_TYPE_PIPE: - # socket or named pipe - return make_stat_result((win32traits._S_IFIFO, - 0, 0, 0, 0, 0, - 0, 0, 0, 0)) - elif filetype == win32traits.FILE_TYPE_UNKNOWN: - error = rwin32.GetLastError_saved() - if error != 0: - raise WindowsError(error, "os_fstat failed") - # else: unknown but valid file + else: + handle = rwin32.get_osfhandle(fd) + win32traits = make_win32_traits(string_traits) + filetype = win32traits.GetFileType(handle) + if filetype == win32traits.FILE_TYPE_CHAR: + # console or LPT device + return make_stat_result((win32traits._S_IFCHR, + 0, 0, 0, 0, 0, + 0, 0, 0, 0)) + elif filetype == win32traits.FILE_TYPE_PIPE: + # socket or named pipe + return make_stat_result((win32traits._S_IFIFO, + 0, 0, 0, 0, 0, + 0, 0, 0, 0)) + elif filetype == win32traits.FILE_TYPE_UNKNOWN: + error = rwin32.GetLastError_saved() + if error != 0: + raise WindowsError(error, "os_fstat failed") + # else: unknown but valid file - # normal disk file (FILE_TYPE_DISK) - info = lltype.malloc(win32traits.BY_HANDLE_FILE_INFORMATION, - flavor='raw', zero=True) - try: - res = win32traits.GetFileInformationByHandle(handle, info) - if res == 0: - raise WindowsError(rwin32.GetLastError_saved(), - "os_fstat failed") - return win32_by_handle_info_to_stat(info) - finally: - lltype.free(info, flavor='raw') + # normal disk file (FILE_TYPE_DISK) + info = lltype.malloc(win32traits.BY_HANDLE_FILE_INFORMATION, + flavor='raw', zero=True) + try: + res = win32traits.GetFileInformationByHandle(handle, info) + if res == 0: + raise WindowsError(rwin32.GetLastError_saved(), + "os_fstat failed") + return win32_by_handle_info_to_stat(win32traits, info) + finally: + lltype.free(info, flavor='raw') @replace_os_function('stat') @specialize.argtype(0) def stat(path): - with lltype.scoped_alloc(STAT_STRUCT.TO) as stresult: - if not _WIN32: + if not _WIN32: + with lltype.scoped_alloc(STAT_STRUCT.TO) as stresult: arg = _as_bytes0(path) handle_posix_error('stat', c_stat(arg, stresult)) return build_stat_result(stresult) - else: - traits = _preferred_traits(path) - path = traits._as_str0(path) - return win32_xstat(path, traverse=True) + else: + traits = _preferred_traits(path) + path = traits.as_str0(path) + return win32_xstat(traits, path, traverse=True) @replace_os_function('lstat') @specialize.argtype(0) def lstat(path): - with lltype.scoped_alloc(STAT_STRUCT.TO) as stresult: - if not _WIN32: + if not _WIN32: + with lltype.scoped_alloc(STAT_STRUCT.TO) as stresult: arg = _as_bytes0(path) handle_posix_error('lstat', c_stat(arg, stresult)) - else: - traits = _preferred_traits(path) - arg = traits._as_str0(path) - handle_posix_error('lstat', c_stat(arg, stresult)) - return build_stat_result(stresult) + else: + traits = _preferred_traits(path) + path = traits.as_str0(path) + return win32_xstat(traits, path, traverse=False) @replace_os_function('fstatvfs') def fstatvfs(fd): @@ -434,6 +441,7 @@ # Seconds between 1.1.1601 and 1.1.1970 secs_between_epochs = rffi.r_longlong(11644473600) + @specialize.arg(0) def win32_xstat(traits, path, traverse=False): win32traits = make_win32_traits(traits) with lltype.scoped_alloc( @@ -443,13 +451,15 @@ if res == 0: errcode = rwin32.GetLastError_saved() if errcode == win32traits.ERROR_SHARING_VIOLATION: - res = win32_attributes_from_dir(path, data) + res = win32_attributes_from_dir( + win32traits, path, data) if res == 0: errcode = rwin32.GetLastError_saved() raise WindowsError(errcode, "os_stat failed") - return attribute_data_to_stat(data) + return win32_attribute_data_to_stat(win32traits, data) - def win32_attributes_to_mode(attributes): + @specialize.arg(0) + def win32_attributes_to_mode(win32traits, attributes): m = 0 attributes = intmask(attributes) if attributes & win32traits.FILE_ATTRIBUTE_DIRECTORY: @@ -462,8 +472,9 @@ m |= 0666 return m - def win32_attribute_data_to_stat(info): - st_mode = win32_attributes_to_mode(info.c_dwFileAttributes) + @specialize.arg(0) + def win32_attribute_data_to_stat(win32traits, info): + st_mode = win32_attributes_to_mode(win32traits, info.c_dwFileAttributes) st_size = make_longlong(info.c_nFileSizeHigh, info.c_nFileSizeLow) ctime = FILE_TIME_to_time_t_float(info.c_ftCreationTime) mtime = FILE_TIME_to_time_t_float(info.c_ftLastWriteTime) @@ -476,9 +487,9 @@ return make_stat_result(result) - def win32_by_handle_info_to_stat(info): + def win32_by_handle_info_to_stat(win32traits, info): # similar to the one above - st_mode = win32_attributes_to_mode(info.c_dwFileAttributes) + st_mode = win32_attributes_to_mode(win32traits, info.c_dwFileAttributes) st_size = make_longlong(info.c_nFileSizeHigh, info.c_nFileSizeLow) ctime = FILE_TIME_to_time_t_float(info.c_ftCreationTime) mtime = FILE_TIME_to_time_t_float(info.c_ftLastWriteTime) @@ -495,10 +506,11 @@ return make_stat_result(result) - def win32_attributes_from_dir(l_path, data): + @specialize.arg(0) + def win32_attributes_from_dir(win32traits, path, data): filedata = lltype.malloc(win32traits.WIN32_FIND_DATA, flavor='raw') try: - hFindFile = win32traits.FindFirstFile(l_path, filedata) + hFindFile = win32traits.FindFirstFile(path, filedata) if hFindFile == rwin32.INVALID_HANDLE_VALUE: return 0 win32traits.FindClose(hFindFile) diff --git a/rpython/rtyper/module/support.py b/rpython/rtyper/module/support.py --- a/rpython/rtyper/module/support.py +++ b/rpython/rtyper/module/support.py @@ -117,6 +117,7 @@ return path.as_unicode() @staticmethod + @specialize.argtype(0) def as_str0(path): res = UnicodeTraits.as_str(path) rstring.check_str0(res) From noreply at buildbot.pypy.org Mon Apr 6 23:25:16 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 6 Apr 2015 23:25:16 +0200 (CEST) Subject: [pypy-commit] pypy more-rposix: Fix merge. Message-ID: <20150406212516.E22C51C1331@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: more-rposix Changeset: r76731:7e3beaacff39 Date: 2015-04-06 23:25 +0200 http://bitbucket.org/pypy/pypy/changeset/7e3beaacff39/ Log: Fix merge. diff --git a/rpython/rlib/rposix_stat.py b/rpython/rlib/rposix_stat.py --- a/rpython/rlib/rposix_stat.py +++ b/rpython/rlib/rposix_stat.py @@ -411,6 +411,7 @@ with lltype.scoped_alloc(STAT_STRUCT.TO) as stresult: arg = _as_bytes0(path) handle_posix_error('lstat', c_stat(arg, stresult)) + return build_stat_result(stresult) else: traits = _preferred_traits(path) path = traits.as_str0(path) From noreply at buildbot.pypy.org Tue Apr 7 11:26:10 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 7 Apr 2015 11:26:10 +0200 (CEST) Subject: [pypy-commit] pypy default: Pick the first of /usr/include/tcl{, 8.6, 8.5} that exists, and select the Message-ID: <20150407092610.59EB01C11E0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76732:394dd8a2203a Date: 2015-04-07 11:26 +0200 http://bitbucket.org/pypy/pypy/changeset/394dd8a2203a/ Log: Pick the first of /usr/include/tcl{,8.6,8.5} that exists, and select the libraries to link with by appending the same version number. diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py --- a/lib_pypy/_tkinter/tklib.py +++ b/lib_pypy/_tkinter/tklib.py @@ -135,9 +135,12 @@ linklibs = ['tcl', 'tk'] libdirs = [] else: - incdirs=['/usr/include/tcl', '/usr/include/tcl8.6', '/usr/include/tcl8.5'] - linklibs=['tcl', 'tk'] - libdirs = [] + for _ver in ['', '8.6', '8.5', '']: + incdirs = ['/usr/include/tcl' + _ver] + linklibs = ['tcl' + _ver, 'tk' + _ver] + libdirs = [] + if os.path.isdir(incdirs[0]): + break tklib = tkffi.verify(""" #include From noreply at buildbot.pypy.org Tue Apr 7 11:31:19 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 7 Apr 2015 11:31:19 +0200 (CEST) Subject: [pypy-commit] pypy default: oops Message-ID: <20150407093119.965001C11E0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76733:8a6ddf44c445 Date: 2015-04-07 11:31 +0200 http://bitbucket.org/pypy/pypy/changeset/8a6ddf44c445/ Log: oops diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py --- a/lib_pypy/_tkinter/tklib.py +++ b/lib_pypy/_tkinter/tklib.py @@ -1,7 +1,7 @@ # C bindings with libtcl and libtk. from cffi import FFI -import sys +import sys, os tkffi = FFI() From noreply at buildbot.pypy.org Tue Apr 7 11:44:46 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 7 Apr 2015 11:44:46 +0200 (CEST) Subject: [pypy-commit] pypy default: Silence an error message that is expected Message-ID: <20150407094446.0E5DE1C1CD5@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76734:1777aca640dd Date: 2015-04-07 09:44 +0000 http://bitbucket.org/pypy/pypy/changeset/1777aca640dd/ Log: Silence an error message that is expected diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -65,8 +65,11 @@ add --without-{0} option to skip packaging binary CFFI extension.""".format(module) raise MissingDependenciesError(module) -def pypy_runs(pypy_c): - return subprocess.call([str(pypy_c), '-c', 'pass']) == 0 +def pypy_runs(pypy_c, quiet=False): + kwds = {} + if quiet: + kwds['stderr'] = subprocess.PIPE + return subprocess.call([str(pypy_c), '-c', 'pass'], **kwds) == 0 def create_package(basedir, options): retval = 0 @@ -108,7 +111,7 @@ # check that this libpypy_c is really needed os.rename(str(libpypy_c), str(libpypy_c) + '~') try: - if pypy_runs(pypy_c): + if pypy_runs(pypy_c, quiet=True): raise Exception("It seems that %r runs without needing %r. " "Please check and remove the latter" % (str(pypy_c), str(libpypy_c))) From noreply at buildbot.pypy.org Tue Apr 7 17:37:40 2015 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 7 Apr 2015 17:37:40 +0200 (CEST) Subject: [pypy-commit] pypy object-dtype2: fix tests Message-ID: <20150407153740.4FDE81C11B5@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: object-dtype2 Changeset: r76735:cf2a8ac882f6 Date: 2015-04-07 09:33 +0300 http://bitbucket.org/pypy/pypy/changeset/cf2a8ac882f6/ Log: fix tests diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -1692,6 +1692,8 @@ w_obj = self.space.newint(w_obj) elif isinstance(w_obj, lltype.Number): w_obj = self.space.newint(w_obj) + elif isinstance(w_obj, float): + w_obj = self.space.newfloat(w_obj) else: raise oefmt(self.space.w_NotImplementedError, "cannot create object array/scalar from lltype") @@ -1718,13 +1720,13 @@ def eq(self, v1, v2): return self.space.eq_w(v1, v2) - @raw_binary_op + @simple_binary_op def max(self, v1, v2): if self.space.is_true(self.space.ge(v1, v2)): return v1 return v2 - @raw_binary_op + @simple_binary_op def min(self, v1, v2): if self.space.is_true(self.space.le(v1, v2)): return v1 @@ -1740,7 +1742,6 @@ not self.space.len_w(v) == 0 def _bool(self, v): - #assert isinstance(v, W_Root) return self.space.bool_w(v) @raw_binary_op From noreply at buildbot.pypy.org Tue Apr 7 19:51:34 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Tue, 7 Apr 2015 19:51:34 +0200 (CEST) Subject: [pypy-commit] pypy refactor-pycall: close branch before merging Message-ID: <20150407175134.E58F81C0204@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: refactor-pycall Changeset: r76736:4709f9c51718 Date: 2015-04-07 18:37 +0100 http://bitbucket.org/pypy/pypy/changeset/4709f9c51718/ Log: close branch before merging From noreply at buildbot.pypy.org Tue Apr 7 19:51:36 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Tue, 7 Apr 2015 19:51:36 +0200 (CEST) Subject: [pypy-commit] pypy default: merge branch refactor-pycall Message-ID: <20150407175136.7C38A1C0204@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76737:cacbf7ed6310 Date: 2015-04-07 18:38 +0100 http://bitbucket.org/pypy/pypy/changeset/cacbf7ed6310/ Log: merge branch refactor-pycall diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py --- a/rpython/annotator/annrpython.py +++ b/rpython/annotator/annrpython.py @@ -10,7 +10,6 @@ Variable, Constant, FunctionGraph, checkgraph) from rpython.translator import simplify, transform from rpython.annotator import model as annmodel, signature -from rpython.annotator.argument import simple_args from rpython.annotator.bookkeeper import Bookkeeper from rpython.rtyper.normalizecalls import perform_normalizations @@ -91,22 +90,14 @@ def get_call_parameters(self, function, args_s, policy): desc = self.bookkeeper.getdesc(function) - args = simple_args(args_s) - result = [] - def schedule(graph, inputcells): - result.append((graph, inputcells)) - return annmodel.s_ImpossibleValue - prevpolicy = self.policy self.policy = policy self.bookkeeper.enter(None) try: - desc.pycall(schedule, args, annmodel.s_ImpossibleValue) + return desc.get_call_parameters(args_s) finally: self.bookkeeper.leave() self.policy = prevpolicy - [(graph, inputcells)] = result - return graph, inputcells def annotate_helper(self, function, args_s, policy=None): if policy is None: diff --git a/rpython/annotator/argument.py b/rpython/annotator/argument.py --- a/rpython/annotator/argument.py +++ b/rpython/annotator/argument.py @@ -155,18 +155,6 @@ keywords_w = [_kwds_w[key] for key in self.keywords] return ArgumentsForTranslation(args_w, dict(zip(self.keywords, keywords_w))) - @classmethod - def fromshape(cls, (shape_cnt, shape_keys, shape_star), data_w): - args_w = data_w[:shape_cnt] - p = end_keys = shape_cnt + len(shape_keys) - if shape_star: - w_star = data_w[p] - p += 1 - else: - w_star = None - return cls(args_w, dict(zip(shape_keys, data_w[shape_cnt:end_keys])), - w_star) - def rawshape(args): return args._rawshape() diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py --- a/rpython/annotator/description.py +++ b/rpython/annotator/description.py @@ -1,13 +1,14 @@ from __future__ import absolute_import import types from rpython.annotator.signature import ( - enforce_signature_args, enforce_signature_return) + enforce_signature_args, enforce_signature_return, finish_type) from rpython.flowspace.model import Constant, FunctionGraph from rpython.flowspace.bytecode import cpython_code_signature -from rpython.annotator.argument import rawshape, ArgErr +from rpython.annotator.argument import rawshape, ArgErr, simple_args from rpython.tool.sourcetools import valid_identifier, func_with_new_name from rpython.tool.pairtype import extendabletype -from rpython.annotator.model import AnnotatorError, SomeInteger, SomeString +from rpython.annotator.model import ( + AnnotatorError, SomeInteger, SomeString, s_ImpossibleValue) class CallFamily(object): """A family of Desc objects that could be called from common call sites. @@ -75,7 +76,6 @@ try: return self.attrs[attrname] except KeyError: - from rpython.annotator.model import s_ImpossibleValue return s_ImpossibleValue def set_s_value(self, attrname, s_value): @@ -97,7 +97,6 @@ # ClassAttrFamily is more precise: it is only about one attribut name. def __init__(self, desc): - from rpython.annotator.model import s_ImpossibleValue self.descs = {desc: True} self.read_locations = {} # set of position_keys self.s_value = s_ImpossibleValue # union of possible values @@ -321,6 +320,24 @@ result = unionof(result, s_previous_result) return result + def get_call_parameters(self, args_s): + args = simple_args(args_s) + inputcells = self.parse_arguments(args) + graph = self.specialize(inputcells) + assert isinstance(graph, FunctionGraph) + # if that graph has a different signature, we need to re-parse + # the arguments. + # recreate the args object because inputcells may have been changed + new_args = args.unmatch_signature(self.signature, inputcells) + inputcells = self.parse_arguments(new_args, graph) + signature = getattr(self.pyobj, '_signature_', None) + if signature: + s_result = finish_type(signature[1], self.bookkeeper, self.pyobj) + if s_result is not None: + self.bookkeeper.annotator.addpendingblock( + graph, graph.returnblock, [s_result]) + return graph, inputcells + def bind_under(self, classdef, name): # XXX static methods return self.bookkeeper.getmethoddesc(self, @@ -352,7 +369,6 @@ @staticmethod def row_to_consider(descs, args, op): # see comments in CallFamily - from rpython.annotator.model import s_ImpossibleValue row = {} for desc in descs: def enlist(graph, ignore): @@ -685,7 +701,6 @@ # look up an attribute in the class cdesc = self.lookup(name) if cdesc is None: - from rpython.annotator.model import s_ImpossibleValue return s_ImpossibleValue else: # delegate to s_get_value to turn it into an annotation @@ -999,7 +1014,6 @@ try: value = self.read_attribute(attr) except AttributeError: - from rpython.annotator.model import s_ImpossibleValue return s_ImpossibleValue else: return self.bookkeeper.immutablevalue(value) diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py --- a/rpython/annotator/test/test_annrpython.py +++ b/rpython/annotator/test/test_annrpython.py @@ -354,6 +354,25 @@ assert isinstance(s, annmodel.SomeInteger) assert s.const == 3 + def test_star_unpack_list(self): + def g(): + pass + def f(l): + return g(*l) + a = self.RPythonAnnotator() + with py.test.raises(annmodel.AnnotatorError): + a.build_types(f, [[int]]) + + def test_star_unpack_and_keywords(self): + def g(a, b, c=0, d=0): + return a + b + c + d + + def f(a, b): + return g(a, *(b,), d=5) + a = self.RPythonAnnotator() + s_result = a.build_types(f, [int, int]) + assert isinstance(s_result, annmodel.SomeInteger) + def test_pbc_attr_preserved_on_instance(self): a = self.RPythonAnnotator() s = a.build_types(snippet.preserve_pbc_attr_on_instance, [bool]) diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py --- a/rpython/annotator/unaryop.py +++ b/rpython/annotator/unaryop.py @@ -6,6 +6,7 @@ from rpython.flowspace.operation import op from rpython.flowspace.model import const, Constant +from rpython.flowspace.argument import CallSpec from rpython.annotator.model import (SomeObject, SomeInteger, SomeBool, SomeString, SomeChar, SomeList, SomeDict, SomeTuple, SomeImpossibleValue, SomeUnicodeCodePoint, SomeInstance, SomeBuiltin, SomeBuiltinMethod, @@ -47,11 +48,35 @@ @op.simple_call.register(SomeObject) def simple_call_SomeObject(annotator, func, *args): - return annotator.annotation(func).call(simple_args([annotator.annotation(arg) for arg in args])) + return annotator.annotation(func).call( + simple_args([annotator.annotation(arg) for arg in args])) + + at op.call_args.register_transform(SomeObject) +def transform_varargs(annotator, v_func, v_shape, *data_v): + callspec = CallSpec.fromshape(v_shape.value, list(data_v)) + v_vararg = callspec.w_stararg + if callspec.w_stararg: + s_vararg = annotator.annotation(callspec.w_stararg) + if not isinstance(s_vararg, SomeTuple): + raise AnnotatorError( + "Calls like f(..., *arg) require 'arg' to be a tuple") + n_items = len(s_vararg.items) + ops = [op.getitem(v_vararg, const(i)) for i in range(n_items)] + new_args = callspec.arguments_w + [hlop.result for hlop in ops] + if callspec.keywords: + newspec = CallSpec(new_args, callspec.keywords) + shape, data_v = newspec.flatten() + call_op = op.call_args(v_func, const(shape), *data_v) + else: + call_op = op.simple_call(v_func, *new_args) + ops.append(call_op) + return ops + @op.call_args.register(SomeObject) -def call_args(annotator, func, *args): - return annotator.annotation(func).call(complex_args([annotator.annotation(arg) for arg in args])) +def call_args(annotator, func, *args_v): + callspec = complex_args([annotator.annotation(v_arg) for v_arg in args_v]) + return annotator.annotation(func).call(callspec) class __extend__(SomeObject): @@ -722,7 +747,7 @@ if attr not in dct: continue obj = dct[attr] - if (not isinstance(obj, Constant) or + if (not isinstance(obj, Constant) or not isinstance(obj.value, property)): return result.append(getattr(obj.value, meth)) diff --git a/rpython/flowspace/argument.py b/rpython/flowspace/argument.py --- a/rpython/flowspace/argument.py +++ b/rpython/flowspace/argument.py @@ -111,3 +111,16 @@ return self.arguments_w else: return self.arguments_w + [const(x) for x in self.w_stararg.value] + + @classmethod + def fromshape(cls, (shape_cnt, shape_keys, shape_star), data_w): + args_w = data_w[:shape_cnt] + p = end_keys = shape_cnt + len(shape_keys) + if shape_star: + w_star = data_w[p] + p += 1 + else: + w_star = None + return cls(args_w, dict(zip(shape_keys, data_w[shape_cnt:end_keys])), + w_star) + diff --git a/rpython/rtyper/normalizecalls.py b/rpython/rtyper/normalizecalls.py --- a/rpython/rtyper/normalizecalls.py +++ b/rpython/rtyper/normalizecalls.py @@ -86,12 +86,7 @@ return False # nothing to do, all signatures already match shape_cnt, shape_keys, shape_star = shape - if shape_star: - raise TyperError( - "not implemented: a call is done with a '*' argument, and the" - " multiple functions or methods that it can go to don't have" - " all the same signature (different argument names or defaults)." - " The call can go to:\n%s" % '\n'.join(map(repr, graphs))) + assert not shape_star, "should have been removed at this stage" # for the first 'shape_cnt' arguments we need to generalize to # a common type diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py --- a/rpython/rtyper/rbuiltin.py +++ b/rpython/rtyper/rbuiltin.py @@ -47,31 +47,14 @@ # to it. return (self.__class__, self.methodname, id(self.s_self)) -def call_args_expand(hop, takes_kwds = True): +def call_args_expand(hop): hop = hop.copy() from rpython.annotator.argument import ArgumentsForTranslation arguments = ArgumentsForTranslation.fromshape( hop.args_s[1].const, # shape range(hop.nb_args-2)) - if arguments.w_stararg is not None: - # expand the *arg in-place -- it must be a tuple - from rpython.rtyper.rtuple import TupleRepr - if arguments.w_stararg != hop.nb_args - 3: - raise TyperError("call pattern too complex") - v_tuple = hop.args_v.pop() - s_tuple = hop.args_s.pop() - r_tuple = hop.args_r.pop() - if not isinstance(r_tuple, TupleRepr): - raise TyperError("*arg must be a tuple") - for i in range(len(r_tuple.items_r)): - v_item = r_tuple.getitem_internal(hop.llops, v_tuple, i) - hop.args_v.append(v_item) - hop.args_s.append(s_tuple.items[i]) - hop.args_r.append(r_tuple.items_r[i]) - + assert arguments.w_stararg is None keywords = arguments.keywords - if not takes_kwds and keywords: - raise TyperError("kwds args not supported") # prefix keyword arguments with 'i_' kwds_i = {} for key in keywords: diff --git a/rpython/rtyper/rptr.py b/rpython/rtyper/rptr.py --- a/rpython/rtyper/rptr.py +++ b/rpython/rtyper/rptr.py @@ -1,4 +1,3 @@ -from rpython.annotator import model as annmodel from rpython.rtyper.llannotation import ( SomePtr, SomeInteriorPtr, SomeLLADTMeth, lltype_to_annotation) from rpython.flowspace import model as flowmodel @@ -107,11 +106,7 @@ resulttype = self.lowleveltype.TO.RESULT) def rtype_call_args(self, hop): - from rpython.rtyper.rbuiltin import call_args_expand - hop, _ = call_args_expand(hop, takes_kwds=False) - hop.swap_fst_snd_args() - hop.r_s_popfirstarg() - return self.rtype_simple_call(hop) + raise TyperError("kwds args not supported") class __extend__(pairtype(PtrRepr, PtrRepr)): def convert_from_to((r_ptr1, r_ptr2), v, llop): diff --git a/rpython/rtyper/test/test_normalizecalls.py b/rpython/rtyper/test/test_normalizecalls.py --- a/rpython/rtyper/test/test_normalizecalls.py +++ b/rpython/rtyper/test/test_normalizecalls.py @@ -237,25 +237,6 @@ import re assert re.match(msg, excinfo.value.args[0]) - def test_methods_with_named_arg_call(self): - class Base: - def fn(self, y): - raise NotImplementedError - class Sub1(Base): - def fn(self, y): - return 1 + y - class Sub2(Base): - def fn(self, x): # different name! - return x - 2 - def dummyfn(n): - if n == 1: - s = Sub1() - else: - s = Sub2() - return s.fn(*(n,)) - - py.test.raises(TyperError, self.rtype, dummyfn, [int], int) - class PBase: def fn(self): From noreply at buildbot.pypy.org Tue Apr 7 19:51:37 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Tue, 7 Apr 2015 19:51:37 +0200 (CEST) Subject: [pypy-commit] pypy default: document branch Message-ID: <20150407175137.A21961C0204@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76738:d78ee80fa1ce Date: 2015-04-07 18:51 +0100 http://bitbucket.org/pypy/pypy/changeset/d78ee80fa1ce/ Log: document branch diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -16,3 +16,8 @@ (incminimark) it no longer needs to create a non-moving copy of it. This saves one `malloc` and copying the data. Additionally a new GC environment variable is introduced (`PYPY_GC_MAX_PINNED`) primarily for debugging purposes. + +.. branch: refactor-pycall +Make `*`-unpacking in RPython function calls completely equivalent to passing +the tuple's elements as arguments. In other words, `f(*(a, b))` now behaves +exactly like `f(a, b)`. From noreply at buildbot.pypy.org Tue Apr 7 20:56:56 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 7 Apr 2015 20:56:56 +0200 (CEST) Subject: [pypy-commit] pypy default: Some more explanations Message-ID: <20150407185656.CEE661C0EEE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76739:8d0c92831d19 Date: 2015-04-07 20:51 +0200 http://bitbucket.org/pypy/pypy/changeset/8d0c92831d19/ Log: Some more explanations diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py --- a/rpython/jit/codewriter/effectinfo.py +++ b/rpython/jit/codewriter/effectinfo.py @@ -126,13 +126,17 @@ if extraeffect == EffectInfo.EF_RANDOM_EFFECTS: assert readonly_descrs_fields is None assert readonly_descrs_arrays is None + assert readonly_descrs_interiorfields is None assert write_descrs_fields is None assert write_descrs_arrays is None + assert write_descrs_interiorfields is None else: assert readonly_descrs_fields is not None assert readonly_descrs_arrays is not None + assert readonly_descrs_interiorfields is not None assert write_descrs_fields is not None assert write_descrs_arrays is not None + assert write_descrs_interiorfields is not None result = object.__new__(cls) result.readonly_descrs_fields = readonly_descrs_fields result.readonly_descrs_arrays = readonly_descrs_arrays @@ -140,6 +144,8 @@ if extraeffect == EffectInfo.EF_LOOPINVARIANT or \ extraeffect == EffectInfo.EF_ELIDABLE_CANNOT_RAISE or \ extraeffect == EffectInfo.EF_ELIDABLE_CAN_RAISE: + # Ignore the writes. Note that this ignores also writes with + # no corresponding reads (rarely the case, but possible). result.write_descrs_fields = [] result.write_descrs_arrays = [] result.write_descrs_interiorfields = [] From noreply at buildbot.pypy.org Tue Apr 7 20:56:58 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 7 Apr 2015 20:56:58 +0200 (CEST) Subject: [pypy-commit] pypy default: Actually use EF_ELIDABLE_CANNOT_RAISE instead of just EF_CANNOT_RAISE Message-ID: <20150407185658.14A5D1C0EEE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76740:d9afe14d9109 Date: 2015-04-07 20:54 +0200 http://bitbucket.org/pypy/pypy/changeset/d9afe14d9109/ Log: Actually use EF_ELIDABLE_CANNOT_RAISE instead of just EF_CANNOT_RAISE for the 'call_pure', which is more in line with what pyjitpl generates. diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -3605,7 +3605,7 @@ ops = ''' [p1, i1, i4] setfield_gc(p1, i1, descr=valuedescr) - i3 = call_pure(p1, descr=cannotraisecalldescr) + i3 = call_pure(p1, descr=elidablecalldescr) setfield_gc(p1, i3, descr=valuedescr) jump(p1, i4, i3) ''' @@ -3617,7 +3617,7 @@ preamble = ''' [p1, i1, i4] setfield_gc(p1, i1, descr=valuedescr) - i3 = call(p1, descr=cannotraisecalldescr) + i3 = call(p1, descr=elidablecalldescr) setfield_gc(p1, i3, descr=valuedescr) i148 = same_as(i3) i147 = same_as(i3) @@ -3630,7 +3630,7 @@ ops = ''' [p1, i1, i4] setfield_gc(p1, i1, descr=valuedescr) - i3 = call_pure(p1, descr=cannotraisecalldescr) + i3 = call_pure(p1, descr=elidablecalldescr) setfield_gc(p1, i1, descr=valuedescr) jump(p1, i4, i3) ''' @@ -3642,8 +3642,7 @@ preamble = ''' [p1, i1, i4] setfield_gc(p1, i1, descr=valuedescr) - i3 = call(p1, descr=cannotraisecalldescr) - setfield_gc(p1, i1, descr=valuedescr) + i3 = call(p1, descr=elidablecalldescr) i151 = same_as(i3) jump(p1, i4, i3, i151) ''' @@ -3656,15 +3655,15 @@ [i0, i1, i2] escape(i1) escape(i2) - i3 = call_pure(123456, 4, 5, 6, descr=cannotraisecalldescr) - i4 = call_pure(123456, 4, i0, 6, descr=cannotraisecalldescr) + i3 = call_pure(123456, 4, 5, 6, descr=elidablecalldescr) + i4 = call_pure(123456, 4, i0, 6, descr=elidablecalldescr) jump(i0, i3, i4) ''' preamble = ''' [i0, i1, i2] escape(i1) escape(i2) - i4 = call(123456, 4, i0, 6, descr=cannotraisecalldescr) + i4 = call(123456, 4, i0, 6, descr=elidablecalldescr) i153 = same_as(i4) jump(i0, i4, i153) ''' diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py --- a/rpython/jit/metainterp/optimizeopt/test/test_util.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py @@ -182,9 +182,10 @@ FUNC = lltype.FuncType([lltype.Signed], lltype.Signed) plaincalldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo.MOST_GENERAL) - cannotraisecalldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [], [], [valuedescr], [], [], - EffectInfo.EF_CANNOT_RAISE)) + elidablecalldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, + EffectInfo([valuedescr], [], [], + [valuedescr], [], [], + EffectInfo.EF_ELIDABLE_CANNOT_RAISE)) nonwritedescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo([], [], [], [], [], [])) writeadescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, From noreply at buildbot.pypy.org Tue Apr 7 22:17:38 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 7 Apr 2015 22:17:38 +0200 (CEST) Subject: [pypy-commit] pypy default: Refine RaiseAnalyzer to be able to distinguish between "real" exceptions Message-ID: <20150407201738.8DD231C1F23@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76741:36a76678c87c Date: 2015-04-07 19:58 +0200 http://bitbucket.org/pypy/pypy/changeset/36a76678c87c/ Log: Refine RaiseAnalyzer to be able to distinguish between "real" exceptions and what is "merely" MemoryErrors. diff --git a/rpython/translator/backendopt/canraise.py b/rpython/translator/backendopt/canraise.py --- a/rpython/translator/backendopt/canraise.py +++ b/rpython/translator/backendopt/canraise.py @@ -9,9 +9,15 @@ class RaiseAnalyzer(graphanalyze.BoolGraphAnalyzer): + ignore_exact_class = None + + def do_ignore_memory_error(self): + self.ignore_exact_class = MemoryError + def analyze_simple_operation(self, op, graphinfo): try: - return bool(LL_OPERATIONS[op.opname].canraise) + canraise = LL_OPERATIONS[op.opname].canraise + return bool(canraise) and canraise != (self.ignore_exact_class,) except KeyError: log.WARNING("Unknown operation: %s" % op.opname) return True @@ -20,7 +26,22 @@ fnobj = op.args[0].value._obj return getattr(fnobj, 'canraise', True) - def analyze_exceptblock(self, block, seen=None): + analyze_exceptblock = None # don't call this + + def analyze_exceptblock_in_graph(self, graph, block, seen=None): + if self.ignore_exact_class is not None: + from rpython.translator.backendopt.ssa import DataFlowFamilyBuilder + dff = DataFlowFamilyBuilder(graph) + variable_families = dff.get_variable_families() + v_exc_instance = variable_families.find_rep(block.inputargs[1]) + for link1 in graph.iterlinks(): + v = link1.last_exc_value + if v is not None: + if variable_families.find_rep(v) is v_exc_instance: + # this is a case of re-raise the exception caught; + # it doesn't count. We'll see the place that really + # raises the exception in the first place. + return False return True # backward compatible interface diff --git a/rpython/translator/backendopt/graphanalyze.py b/rpython/translator/backendopt/graphanalyze.py --- a/rpython/translator/backendopt/graphanalyze.py +++ b/rpython/translator/backendopt/graphanalyze.py @@ -49,6 +49,9 @@ def analyze_exceptblock(self, block, seen=None): return self.bottom_result() + def analyze_exceptblock_in_graph(self, graph, block, seen=None): + return self.analyze_exceptblock(block, seen) + def analyze_startblock(self, block, seen=None): return self.bottom_result() @@ -121,7 +124,7 @@ elif block is graph.exceptblock: result = self.add_to_result( result, - self.analyze_exceptblock(block, seen) + self.analyze_exceptblock_in_graph(graph, block, seen) ) if not self.is_top_result(result): for op in block.operations: diff --git a/rpython/translator/backendopt/test/test_canraise.py b/rpython/translator/backendopt/test/test_canraise.py --- a/rpython/translator/backendopt/test/test_canraise.py +++ b/rpython/translator/backendopt/test/test_canraise.py @@ -225,3 +225,32 @@ fgraph = graphof(t, f) result = ra.can_raise(fgraph.startblock.operations[0]) assert not result + + def test_memoryerror(self): + def f(x): + return [x, 42] + t, ra = self.translate(f, [int]) + result = ra.analyze_direct_call(graphof(t, f)) + assert result + # + ra = RaiseAnalyzer(t) + ra.do_ignore_memory_error() + result = ra.analyze_direct_call(graphof(t, f)) + assert not result + # + def g(x): + try: + return f(x) + except: + raise + t, ra = self.translate(g, [int]) + ra.do_ignore_memory_error() + result = ra.analyze_direct_call(graphof(t, g)) + assert not result + # + def h(x): + return {5:6}[x] + t, ra = self.translate(h, [int]) + ra.do_ignore_memory_error() # but it's potentially a KeyError + result = ra.analyze_direct_call(graphof(t, h)) + assert result From noreply at buildbot.pypy.org Tue Apr 7 22:17:39 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 7 Apr 2015 22:17:39 +0200 (CEST) Subject: [pypy-commit] pypy default: Add a new EF_ELIDABLE_OR_MEMORYERROR for elidable functions Message-ID: <20150407201739.AEB371C1F23@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76742:1c3d074e06b7 Date: 2015-04-07 20:17 +0200 http://bitbucket.org/pypy/pypy/changeset/1c3d074e06b7/ Log: Add a new EF_ELIDABLE_OR_MEMORYERROR for elidable functions that could raise, but only MemoryError diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py --- a/rpython/jit/codewriter/call.py +++ b/rpython/jit/codewriter/call.py @@ -31,6 +31,8 @@ self.rtyper = cpu.rtyper translator = self.rtyper.annotator.translator self.raise_analyzer = RaiseAnalyzer(translator) + self.raise_analyzer_ignore_memoryerror = RaiseAnalyzer(translator) + self.raise_analyzer_ignore_memoryerror.do_ignore_memory_error() self.readwrite_analyzer = ReadWriteAnalyzer(translator) self.virtualizable_analyzer = VirtualizableAnalyzer(translator) self.quasiimmut_analyzer = QuasiImmutAnalyzer(translator) @@ -260,11 +262,14 @@ elif loopinvariant: extraeffect = EffectInfo.EF_LOOPINVARIANT elif elidable: - if self._canraise(op): + cr = self._canraise(op) + if cr == "mem": + extraeffect = EffectInfo.EF_ELIDABLE_OR_MEMORYERROR + elif cr: extraeffect = EffectInfo.EF_ELIDABLE_CAN_RAISE else: extraeffect = EffectInfo.EF_ELIDABLE_CANNOT_RAISE - elif self._canraise(op): + elif self._canraise(op): # True or "mem" extraeffect = EffectInfo.EF_CAN_RAISE else: extraeffect = EffectInfo.EF_CANNOT_RAISE @@ -278,6 +283,7 @@ " effects): EF=%s" % (op, extraeffect)) if elidable: if extraeffect not in (EffectInfo.EF_ELIDABLE_CANNOT_RAISE, + EffectInfo.EF_ELIDABLE_OR_MEMORYERROR, EffectInfo.EF_ELIDABLE_CAN_RAISE): raise Exception( "in operation %r: this calls an _elidable_function_," @@ -301,10 +307,17 @@ effectinfo) def _canraise(self, op): + """Returns True, False, or "mem" to mean 'only MemoryError'.""" if op.opname == 'pseudo_call_cannot_raise': return False try: - return self.raise_analyzer.can_raise(op) + if self.raise_analyzer.can_raise(op): + if self.raise_analyzer_ignore_memoryerror.can_raise(op): + return True + else: + return "mem" + else: + return False except lltype.DelayedPointer: return True # if we need to look into the delayed ptr that is # the portal, then it's certainly going to raise diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py --- a/rpython/jit/codewriter/effectinfo.py +++ b/rpython/jit/codewriter/effectinfo.py @@ -11,10 +11,11 @@ EF_ELIDABLE_CANNOT_RAISE = 0 #elidable function (and cannot raise) EF_LOOPINVARIANT = 1 #special: call it only once per loop EF_CANNOT_RAISE = 2 #a function which cannot raise - EF_ELIDABLE_CAN_RAISE = 3 #elidable function (but can raise) - EF_CAN_RAISE = 4 #normal function (can raise) - EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE = 5 #can raise and force virtualizables - EF_RANDOM_EFFECTS = 6 #can do whatever + EF_ELIDABLE_OR_MEMORYERROR = 3 #elidable, can only raise MemoryError + EF_ELIDABLE_CAN_RAISE = 4 #elidable function (but can raise) + EF_CAN_RAISE = 5 #normal function (can raise) + EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE = 6 #can raise and force virtualizables + EF_RANDOM_EFFECTS = 7 #can do whatever # the 'oopspecindex' field is one of the following values: OS_NONE = 0 # normal case, no oopspec @@ -139,6 +140,7 @@ result.readonly_descrs_interiorfields = readonly_descrs_interiorfields if extraeffect == EffectInfo.EF_LOOPINVARIANT or \ extraeffect == EffectInfo.EF_ELIDABLE_CANNOT_RAISE or \ + extraeffect == EffectInfo.EF_ELIDABLE_OR_MEMORYERROR or \ extraeffect == EffectInfo.EF_ELIDABLE_CAN_RAISE: result.write_descrs_fields = [] result.write_descrs_arrays = [] @@ -152,19 +154,23 @@ result.oopspecindex = oopspecindex result.extradescrs = extradescrs result.call_release_gil_target = call_release_gil_target - if result.check_can_raise(): + if result.check_can_raise(ignore_memoryerror=True): assert oopspecindex in cls._OS_CANRAISE cls._cache[key] = result return result - def check_can_raise(self): - return self.extraeffect > self.EF_CANNOT_RAISE + def check_can_raise(self, ignore_memoryerror=False): + if ignore_memoryerror: + return self.extraeffect > self.EF_ELIDABLE_OR_MEMORYERROR + else: + return self.extraeffect > self.EF_CANNOT_RAISE def check_can_invalidate(self): return self.can_invalidate def check_is_elidable(self): return (self.extraeffect == self.EF_ELIDABLE_CAN_RAISE or + self.extraeffect == self.EF_ELIDABLE_OR_MEMORYERROR or self.extraeffect == self.EF_ELIDABLE_CANNOT_RAISE) def check_forces_virtual_or_virtualizable(self): diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -1847,6 +1847,13 @@ def _handle_stroruni_call(self, op, oopspec_name, args): SoU = args[0].concretetype # Ptr(STR) or Ptr(UNICODE) + can_raise_memoryerror = { + "stroruni.concat": True, + "stroruni.slice": True, + "stroruni.equal": False, + "stroruni.cmp": False, + "stroruni.copy_string_to_raw": False, + } if SoU.TO == rstr.STR: dict = {"stroruni.concat": EffectInfo.OS_STR_CONCAT, "stroruni.slice": EffectInfo.OS_STR_SLICE, @@ -1913,8 +1920,11 @@ argtypes, resulttype, EffectInfo.EF_ELIDABLE_CANNOT_RAISE) # - return self._handle_oopspec_call(op, args, dict[oopspec_name], - EffectInfo.EF_ELIDABLE_CANNOT_RAISE) + if can_raise_memoryerror[oopspec_name]: + extra = EffectInfo.EF_ELIDABLE_OR_MEMORYERROR + else: + extra = EffectInfo.EF_ELIDABLE_CANNOT_RAISE + return self._handle_oopspec_call(op, args, dict[oopspec_name], extra) def _handle_str2unicode_call(self, op, oopspec_name, args): # ll_str2unicode can raise UnicodeDecodeError diff --git a/rpython/jit/codewriter/test/test_call.py b/rpython/jit/codewriter/test/test_call.py --- a/rpython/jit/codewriter/test/test_call.py +++ b/rpython/jit/codewriter/test/test_call.py @@ -6,6 +6,7 @@ from rpython.rlib import jit from rpython.jit.codewriter import support, call from rpython.jit.codewriter.call import CallControl +from rpython.jit.codewriter.effectinfo import EffectInfo class FakePolicy: @@ -279,3 +280,39 @@ call_descr = cc.getcalldescr(op) assert not call_descr.extrainfo.has_random_effects() assert call_descr.extrainfo.check_is_elidable() + +def test_elidable_kinds(): + from rpython.jit.backend.llgraph.runner import LLGraphCPU + + @jit.elidable + def f1(n, m): + return n + m + @jit.elidable + def f2(n, m): + return [n, m] # may raise MemoryError + @jit.elidable + def f3(n, m): + if n > m: + raise ValueError + return n + m + + def f(n, m): + a = f1(n, m) + b = f2(n, m) + c = f3(n, m) + return a + len(b) + c + + rtyper = support.annotate(f, [7, 9]) + jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0]) + cc = CallControl(LLGraphCPU(rtyper), jitdrivers_sd=[jitdriver_sd]) + res = cc.find_all_graphs(FakePolicy()) + [f_graph] = [x for x in res if x.func is f] + + for index, expected in [ + (0, EffectInfo.EF_ELIDABLE_CANNOT_RAISE), + (1, EffectInfo.EF_ELIDABLE_OR_MEMORYERROR), + (2, EffectInfo.EF_ELIDABLE_CAN_RAISE)]: + call_op = f_graph.startblock.operations[index] + assert call_op.opname == 'direct_call' + call_descr = cc.getcalldescr(call_op) + assert call_descr.extrainfo.extraeffect == expected diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py --- a/rpython/jit/codewriter/test/test_jtransform.py +++ b/rpython/jit/codewriter/test/test_jtransform.py @@ -161,6 +161,9 @@ assert extraeffect == EI.EF_CANNOT_RAISE elif oopspecindex == EI.OS_THREADLOCALREF_GET: assert extraeffect == self.expected_effect_of_threadlocalref_get + elif oopspecindex in (EI.OS_STR_CONCAT, EI.OS_UNI_CONCAT, + EI.OS_STR_SLICE, EI.OS_UNI_SLICE): + assert extraeffect == EI.EF_ELIDABLE_OR_MEMORYERROR else: assert extraeffect == EI.EF_ELIDABLE_CANNOT_RAISE return 'calldescr-%d' % oopspecindex From noreply at buildbot.pypy.org Tue Apr 7 22:17:40 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 7 Apr 2015 22:17:40 +0200 (CEST) Subject: [pypy-commit] pypy default: merge heads Message-ID: <20150407201740.E683D1C1F23@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76743:3c8a7c3382b5 Date: 2015-04-07 20:55 +0200 http://bitbucket.org/pypy/pypy/changeset/3c8a7c3382b5/ Log: merge heads diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py --- a/rpython/jit/codewriter/call.py +++ b/rpython/jit/codewriter/call.py @@ -31,6 +31,8 @@ self.rtyper = cpu.rtyper translator = self.rtyper.annotator.translator self.raise_analyzer = RaiseAnalyzer(translator) + self.raise_analyzer_ignore_memoryerror = RaiseAnalyzer(translator) + self.raise_analyzer_ignore_memoryerror.do_ignore_memory_error() self.readwrite_analyzer = ReadWriteAnalyzer(translator) self.virtualizable_analyzer = VirtualizableAnalyzer(translator) self.quasiimmut_analyzer = QuasiImmutAnalyzer(translator) @@ -260,11 +262,14 @@ elif loopinvariant: extraeffect = EffectInfo.EF_LOOPINVARIANT elif elidable: - if self._canraise(op): + cr = self._canraise(op) + if cr == "mem": + extraeffect = EffectInfo.EF_ELIDABLE_OR_MEMORYERROR + elif cr: extraeffect = EffectInfo.EF_ELIDABLE_CAN_RAISE else: extraeffect = EffectInfo.EF_ELIDABLE_CANNOT_RAISE - elif self._canraise(op): + elif self._canraise(op): # True or "mem" extraeffect = EffectInfo.EF_CAN_RAISE else: extraeffect = EffectInfo.EF_CANNOT_RAISE @@ -278,6 +283,7 @@ " effects): EF=%s" % (op, extraeffect)) if elidable: if extraeffect not in (EffectInfo.EF_ELIDABLE_CANNOT_RAISE, + EffectInfo.EF_ELIDABLE_OR_MEMORYERROR, EffectInfo.EF_ELIDABLE_CAN_RAISE): raise Exception( "in operation %r: this calls an _elidable_function_," @@ -301,10 +307,17 @@ effectinfo) def _canraise(self, op): + """Returns True, False, or "mem" to mean 'only MemoryError'.""" if op.opname == 'pseudo_call_cannot_raise': return False try: - return self.raise_analyzer.can_raise(op) + if self.raise_analyzer.can_raise(op): + if self.raise_analyzer_ignore_memoryerror.can_raise(op): + return True + else: + return "mem" + else: + return False except lltype.DelayedPointer: return True # if we need to look into the delayed ptr that is # the portal, then it's certainly going to raise diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py --- a/rpython/jit/codewriter/effectinfo.py +++ b/rpython/jit/codewriter/effectinfo.py @@ -11,10 +11,11 @@ EF_ELIDABLE_CANNOT_RAISE = 0 #elidable function (and cannot raise) EF_LOOPINVARIANT = 1 #special: call it only once per loop EF_CANNOT_RAISE = 2 #a function which cannot raise - EF_ELIDABLE_CAN_RAISE = 3 #elidable function (but can raise) - EF_CAN_RAISE = 4 #normal function (can raise) - EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE = 5 #can raise and force virtualizables - EF_RANDOM_EFFECTS = 6 #can do whatever + EF_ELIDABLE_OR_MEMORYERROR = 3 #elidable, can only raise MemoryError + EF_ELIDABLE_CAN_RAISE = 4 #elidable function (but can raise) + EF_CAN_RAISE = 5 #normal function (can raise) + EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE = 6 #can raise and force virtualizables + EF_RANDOM_EFFECTS = 7 #can do whatever # the 'oopspecindex' field is one of the following values: OS_NONE = 0 # normal case, no oopspec @@ -143,6 +144,7 @@ result.readonly_descrs_interiorfields = readonly_descrs_interiorfields if extraeffect == EffectInfo.EF_LOOPINVARIANT or \ extraeffect == EffectInfo.EF_ELIDABLE_CANNOT_RAISE or \ + extraeffect == EffectInfo.EF_ELIDABLE_OR_MEMORYERROR or \ extraeffect == EffectInfo.EF_ELIDABLE_CAN_RAISE: # Ignore the writes. Note that this ignores also writes with # no corresponding reads (rarely the case, but possible). @@ -158,19 +160,23 @@ result.oopspecindex = oopspecindex result.extradescrs = extradescrs result.call_release_gil_target = call_release_gil_target - if result.check_can_raise(): + if result.check_can_raise(ignore_memoryerror=True): assert oopspecindex in cls._OS_CANRAISE cls._cache[key] = result return result - def check_can_raise(self): - return self.extraeffect > self.EF_CANNOT_RAISE + def check_can_raise(self, ignore_memoryerror=False): + if ignore_memoryerror: + return self.extraeffect > self.EF_ELIDABLE_OR_MEMORYERROR + else: + return self.extraeffect > self.EF_CANNOT_RAISE def check_can_invalidate(self): return self.can_invalidate def check_is_elidable(self): return (self.extraeffect == self.EF_ELIDABLE_CAN_RAISE or + self.extraeffect == self.EF_ELIDABLE_OR_MEMORYERROR or self.extraeffect == self.EF_ELIDABLE_CANNOT_RAISE) def check_forces_virtual_or_virtualizable(self): diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -1847,6 +1847,13 @@ def _handle_stroruni_call(self, op, oopspec_name, args): SoU = args[0].concretetype # Ptr(STR) or Ptr(UNICODE) + can_raise_memoryerror = { + "stroruni.concat": True, + "stroruni.slice": True, + "stroruni.equal": False, + "stroruni.cmp": False, + "stroruni.copy_string_to_raw": False, + } if SoU.TO == rstr.STR: dict = {"stroruni.concat": EffectInfo.OS_STR_CONCAT, "stroruni.slice": EffectInfo.OS_STR_SLICE, @@ -1913,8 +1920,11 @@ argtypes, resulttype, EffectInfo.EF_ELIDABLE_CANNOT_RAISE) # - return self._handle_oopspec_call(op, args, dict[oopspec_name], - EffectInfo.EF_ELIDABLE_CANNOT_RAISE) + if can_raise_memoryerror[oopspec_name]: + extra = EffectInfo.EF_ELIDABLE_OR_MEMORYERROR + else: + extra = EffectInfo.EF_ELIDABLE_CANNOT_RAISE + return self._handle_oopspec_call(op, args, dict[oopspec_name], extra) def _handle_str2unicode_call(self, op, oopspec_name, args): # ll_str2unicode can raise UnicodeDecodeError diff --git a/rpython/jit/codewriter/test/test_call.py b/rpython/jit/codewriter/test/test_call.py --- a/rpython/jit/codewriter/test/test_call.py +++ b/rpython/jit/codewriter/test/test_call.py @@ -6,6 +6,7 @@ from rpython.rlib import jit from rpython.jit.codewriter import support, call from rpython.jit.codewriter.call import CallControl +from rpython.jit.codewriter.effectinfo import EffectInfo class FakePolicy: @@ -279,3 +280,39 @@ call_descr = cc.getcalldescr(op) assert not call_descr.extrainfo.has_random_effects() assert call_descr.extrainfo.check_is_elidable() + +def test_elidable_kinds(): + from rpython.jit.backend.llgraph.runner import LLGraphCPU + + @jit.elidable + def f1(n, m): + return n + m + @jit.elidable + def f2(n, m): + return [n, m] # may raise MemoryError + @jit.elidable + def f3(n, m): + if n > m: + raise ValueError + return n + m + + def f(n, m): + a = f1(n, m) + b = f2(n, m) + c = f3(n, m) + return a + len(b) + c + + rtyper = support.annotate(f, [7, 9]) + jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0]) + cc = CallControl(LLGraphCPU(rtyper), jitdrivers_sd=[jitdriver_sd]) + res = cc.find_all_graphs(FakePolicy()) + [f_graph] = [x for x in res if x.func is f] + + for index, expected in [ + (0, EffectInfo.EF_ELIDABLE_CANNOT_RAISE), + (1, EffectInfo.EF_ELIDABLE_OR_MEMORYERROR), + (2, EffectInfo.EF_ELIDABLE_CAN_RAISE)]: + call_op = f_graph.startblock.operations[index] + assert call_op.opname == 'direct_call' + call_descr = cc.getcalldescr(call_op) + assert call_descr.extrainfo.extraeffect == expected diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py --- a/rpython/jit/codewriter/test/test_jtransform.py +++ b/rpython/jit/codewriter/test/test_jtransform.py @@ -161,6 +161,9 @@ assert extraeffect == EI.EF_CANNOT_RAISE elif oopspecindex == EI.OS_THREADLOCALREF_GET: assert extraeffect == self.expected_effect_of_threadlocalref_get + elif oopspecindex in (EI.OS_STR_CONCAT, EI.OS_UNI_CONCAT, + EI.OS_STR_SLICE, EI.OS_UNI_SLICE): + assert extraeffect == EI.EF_ELIDABLE_OR_MEMORYERROR else: assert extraeffect == EI.EF_ELIDABLE_CANNOT_RAISE return 'calldescr-%d' % oopspecindex diff --git a/rpython/translator/backendopt/canraise.py b/rpython/translator/backendopt/canraise.py --- a/rpython/translator/backendopt/canraise.py +++ b/rpython/translator/backendopt/canraise.py @@ -9,9 +9,15 @@ class RaiseAnalyzer(graphanalyze.BoolGraphAnalyzer): + ignore_exact_class = None + + def do_ignore_memory_error(self): + self.ignore_exact_class = MemoryError + def analyze_simple_operation(self, op, graphinfo): try: - return bool(LL_OPERATIONS[op.opname].canraise) + canraise = LL_OPERATIONS[op.opname].canraise + return bool(canraise) and canraise != (self.ignore_exact_class,) except KeyError: log.WARNING("Unknown operation: %s" % op.opname) return True @@ -20,7 +26,22 @@ fnobj = op.args[0].value._obj return getattr(fnobj, 'canraise', True) - def analyze_exceptblock(self, block, seen=None): + analyze_exceptblock = None # don't call this + + def analyze_exceptblock_in_graph(self, graph, block, seen=None): + if self.ignore_exact_class is not None: + from rpython.translator.backendopt.ssa import DataFlowFamilyBuilder + dff = DataFlowFamilyBuilder(graph) + variable_families = dff.get_variable_families() + v_exc_instance = variable_families.find_rep(block.inputargs[1]) + for link1 in graph.iterlinks(): + v = link1.last_exc_value + if v is not None: + if variable_families.find_rep(v) is v_exc_instance: + # this is a case of re-raise the exception caught; + # it doesn't count. We'll see the place that really + # raises the exception in the first place. + return False return True # backward compatible interface diff --git a/rpython/translator/backendopt/graphanalyze.py b/rpython/translator/backendopt/graphanalyze.py --- a/rpython/translator/backendopt/graphanalyze.py +++ b/rpython/translator/backendopt/graphanalyze.py @@ -49,6 +49,9 @@ def analyze_exceptblock(self, block, seen=None): return self.bottom_result() + def analyze_exceptblock_in_graph(self, graph, block, seen=None): + return self.analyze_exceptblock(block, seen) + def analyze_startblock(self, block, seen=None): return self.bottom_result() @@ -121,7 +124,7 @@ elif block is graph.exceptblock: result = self.add_to_result( result, - self.analyze_exceptblock(block, seen) + self.analyze_exceptblock_in_graph(graph, block, seen) ) if not self.is_top_result(result): for op in block.operations: diff --git a/rpython/translator/backendopt/test/test_canraise.py b/rpython/translator/backendopt/test/test_canraise.py --- a/rpython/translator/backendopt/test/test_canraise.py +++ b/rpython/translator/backendopt/test/test_canraise.py @@ -225,3 +225,32 @@ fgraph = graphof(t, f) result = ra.can_raise(fgraph.startblock.operations[0]) assert not result + + def test_memoryerror(self): + def f(x): + return [x, 42] + t, ra = self.translate(f, [int]) + result = ra.analyze_direct_call(graphof(t, f)) + assert result + # + ra = RaiseAnalyzer(t) + ra.do_ignore_memory_error() + result = ra.analyze_direct_call(graphof(t, f)) + assert not result + # + def g(x): + try: + return f(x) + except: + raise + t, ra = self.translate(g, [int]) + ra.do_ignore_memory_error() + result = ra.analyze_direct_call(graphof(t, g)) + assert not result + # + def h(x): + return {5:6}[x] + t, ra = self.translate(h, [int]) + ra.do_ignore_memory_error() # but it's potentially a KeyError + result = ra.analyze_direct_call(graphof(t, h)) + assert result From noreply at buildbot.pypy.org Tue Apr 7 22:17:42 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 7 Apr 2015 22:17:42 +0200 (CEST) Subject: [pypy-commit] pypy default: The EF_ELIDABLE_OR_MEMORYERROR functions can also be moved out of loops. Message-ID: <20150407201742.20A131C1F23@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76744:cdbda6639c3b Date: 2015-04-07 22:17 +0200 http://bitbucket.org/pypy/pypy/changeset/cdbda6639c3b/ Log: The EF_ELIDABLE_OR_MEMORYERROR functions can also be moved out of loops. It differs from the EF_ELIDABLE_CANNOT_RAISE functions because they still need the guard_no_exception afterward to check for MemoryError; and it differs from the EF_ELIDABLE_CAN_RAISE functions because these ones should still not be removed from loops, as 56eebe9dd813. diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py --- a/rpython/jit/metainterp/heapcache.py +++ b/rpython/jit/metainterp/heapcache.py @@ -191,6 +191,7 @@ ef = effectinfo.extraeffect if (ef == effectinfo.EF_LOOPINVARIANT or ef == effectinfo.EF_ELIDABLE_CANNOT_RAISE or + ef == effectinfo.EF_ELIDABLE_OR_MEMORYERROR or ef == effectinfo.EF_ELIDABLE_CAN_RAISE): return # A special case for ll_arraycopy, because it is so common, and its diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py --- a/rpython/jit/metainterp/optimizeopt/pure.py +++ b/rpython/jit/metainterp/optimizeopt/pure.py @@ -84,7 +84,7 @@ # don't move call_pure_with_exception in the short preamble... # issue #2015 effectinfo = op.getdescr().get_extra_info() - if not effectinfo.check_can_raise(): + if not effectinfo.check_can_raise(ignore_memoryerror=True): self.call_pure_positions.append( len(self.optimizer._newoperations) - 1) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -3625,6 +3625,63 @@ ''' self.optimize_loop(ops, expected, preamble) + def test_call_pure_invalidates_caches_2(self): + # same as test_call_pure_invalidates_caches, but with + # an EF_ELIDABLE_OR_MEMORYERROR, which can still be moved + # out of the main loop potentially into the short preamble + ops = ''' + [p1, i1, i4] + setfield_gc(p1, i1, descr=valuedescr) + i3 = call_pure(p1, descr=elidable2calldescr) + setfield_gc(p1, i3, descr=valuedescr) + jump(p1, i4, i3) + ''' + expected = ''' + [p1, i4, i3, i5] + setfield_gc(p1, i5, descr=valuedescr) + jump(p1, i3, i5, i5) + ''' + preamble = ''' + [p1, i1, i4] + setfield_gc(p1, i1, descr=valuedescr) + i3 = call(p1, descr=elidable2calldescr) + setfield_gc(p1, i3, descr=valuedescr) + i148 = same_as(i3) + i147 = same_as(i3) + jump(p1, i4, i3, i148) + ''' + self.optimize_loop(ops, expected, preamble) + + def test_call_pure_can_raise(self): + # same as test_call_pure_invalidates_caches, but this time + # with an EF_ELIDABLE_CAN_RAISE, which *cannot* be moved + # out of the main loop: the problem is that it cannot be + # present in the short preamble, because nobody would catch + # the potential exception + ops = ''' + [p1, i1, i4] + setfield_gc(p1, i1, descr=valuedescr) + i3 = call_pure(p1, descr=elidable3calldescr) + setfield_gc(p1, i3, descr=valuedescr) + jump(p1, i4, i3) + ''' + expected = ''' + [p1, i1, i4] + setfield_gc(p1, i1, descr=valuedescr) + i3 = call(p1, descr=elidable3calldescr) + setfield_gc(p1, i3, descr=valuedescr) + jump(p1, i4, i3) + ''' + preamble = ''' + [p1, i1, i4] + setfield_gc(p1, i1, descr=valuedescr) + i3 = call(p1, descr=elidable3calldescr) + setfield_gc(p1, i3, descr=valuedescr) + i167 = same_as(i3) + jump(p1, i4, i3) + ''' + self.optimize_loop(ops, expected, preamble) + def test_call_pure_invalidates_heap_knowledge(self): # CALL_PURE should still force the setfield_gc() to occur before it ops = ''' @@ -3677,25 +3734,31 @@ def test_call_pure_constant_folding_exc(self): # CALL_PURE may be followed by GUARD_NO_EXCEPTION - # XXX maybe temporary, but we can't remove such call_pures from - # the loop, because the short preamble can't call them safely. + # we can't remove such call_pures from the loop, + # because the short preamble can't call them safely. + # We can still check that an all-constant call_pure is removed, + # and that duplicate call_pures are folded. arg_consts = [ConstInt(i) for i in (123456, 4, 5, 6)] call_pure_results = {tuple(arg_consts): ConstInt(42)} ops = ''' - [i0, i1, i2] + [i0, i1, i2, i9] escape(i1) escape(i2) - i3 = call_pure(123456, 4, 5, 6, descr=plaincalldescr) + escape(i9) + i3 = call_pure(123456, 4, 5, 6, descr=elidable3calldescr) guard_no_exception() [] - i4 = call_pure(123456, 4, i0, 6, descr=plaincalldescr) + i4 = call_pure(123456, 4, i0, 6, descr=elidable3calldescr) guard_no_exception() [] - jump(i0, i3, i4) + i5 = call_pure(123456, 4, i0, 6, descr=elidable3calldescr) + guard_no_exception() [] + jump(i0, i3, i4, i5) ''' preamble = ''' - [i0, i1, i2] + [i0, i1, i2, i9] escape(i1) escape(i2) - i4 = call(123456, 4, i0, 6, descr=plaincalldescr) + escape(i9) + i4 = call(123456, 4, i0, 6, descr=elidable3calldescr) guard_no_exception() [] jump(i0, i4) ''' @@ -3703,7 +3766,8 @@ [i0, i2] escape(42) escape(i2) - i4 = call(123456, 4, i0, 6, descr=plaincalldescr) + escape(i2) + i4 = call(123456, 4, i0, 6, descr=elidable3calldescr) guard_no_exception() [] jump(i0, i4) ''' diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py --- a/rpython/jit/metainterp/optimizeopt/test/test_util.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py @@ -186,6 +186,14 @@ EffectInfo([valuedescr], [], [], [valuedescr], [], [], EffectInfo.EF_ELIDABLE_CANNOT_RAISE)) + elidable2calldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, + EffectInfo([valuedescr], [], [], + [valuedescr], [], [], + EffectInfo.EF_ELIDABLE_OR_MEMORYERROR)) + elidable3calldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, + EffectInfo([valuedescr], [], [], + [valuedescr], [], [], + EffectInfo.EF_ELIDABLE_CAN_RAISE)) nonwritedescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo([], [], [], [], [], [])) writeadescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, @@ -259,20 +267,24 @@ ('streq_checknull_char_descr', 'OS_STREQ_CHECKNULL_CHAR'), ('streq_lengthok_descr', 'OS_STREQ_LENGTHOK'), ]: + if _name in ('strconcatdescr', 'strslicedescr'): + _extra = EffectInfo.EF_ELIDABLE_OR_MEMORYERROR + else: + _extra = EffectInfo.EF_ELIDABLE_CANNOT_RAISE _oopspecindex = getattr(EffectInfo, _os) locals()[_name] = \ cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [], [], [], [], [], EffectInfo.EF_CANNOT_RAISE, + EffectInfo([], [], [], [], [], [], _extra, oopspecindex=_oopspecindex)) # _oopspecindex = getattr(EffectInfo, _os.replace('STR', 'UNI')) locals()[_name.replace('str', 'unicode')] = \ cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [], [], [], [], [], EffectInfo.EF_CANNOT_RAISE, + EffectInfo([], [], [], [], [], [], _extra, oopspecindex=_oopspecindex)) s2u_descr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [], [], [], [], [], + EffectInfo([], [], [], [], [], [], EffectInfo.EF_ELIDABLE_CAN_RAISE, oopspecindex=EffectInfo.OS_STR2UNICODE)) # diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py --- a/rpython/jit/metainterp/optimizeopt/unroll.py +++ b/rpython/jit/metainterp/optimizeopt/unroll.py @@ -438,7 +438,21 @@ if op.is_ovf(): guard = ResOperation(rop.GUARD_NO_OVERFLOW, [], None) optimizer.send_extra_operation(guard) - assert not op.is_call_pure_with_exception() + if self.is_call_pure_with_exception(op): # only for MemoryError + guard = ResOperation(rop.GUARD_NO_EXCEPTION, [], None) + optimizer.send_extra_operation(guard) + + def is_call_pure_with_exception(self, op): + if op.getopnum() == rop.CALL_PURE: + effectinfo = op.getdescr().get_extra_info() + # Assert that only EF_ELIDABLE_CANNOT_RAISE or + # EF_ELIDABLE_OR_MEMORYERROR end up here, not + # for example EF_ELIDABLE_CAN_RAISE. + assert effectinfo.extraeffect in ( + effectinfo.EF_ELIDABLE_CANNOT_RAISE, + effectinfo.EF_ELIDABLE_OR_MEMORYERROR) + return effectinfo.extraeffect != effectinfo.EF_ELIDABLE_CANNOT_RAISE + return False def add_op_to_short(self, op, emit=True, guards_needed=False): if op is None: @@ -472,7 +486,9 @@ # FIXME: ensure that GUARD_OVERFLOW:ed ops not end up here guard = ResOperation(rop.GUARD_NO_OVERFLOW, [], None) self.add_op_to_short(guard, emit, guards_needed) - assert not op.is_call_pure_with_exception() + if self.is_call_pure_with_exception(op): # only for MemoryError + guard = ResOperation(rop.GUARD_NO_EXCEPTION, [], None) + self.add_op_to_short(guard, emit, guards_needed) for guard in value_guards: self.add_op_to_short(guard, emit, guards_needed) diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -176,12 +176,6 @@ def returns_bool_result(self): return self._cls_has_bool_result - def is_call_pure_with_exception(self): - if self.getopnum() == rop.CALL_PURE: - effectinfo = self.getdescr().get_extra_info() - return effectinfo.check_can_raise() - return False - # =================== # Top of the hierachy diff --git a/rpython/jit/metainterp/test/test_heapcache.py b/rpython/jit/metainterp/test/test_heapcache.py --- a/rpython/jit/metainterp/test/test_heapcache.py +++ b/rpython/jit/metainterp/test/test_heapcache.py @@ -22,10 +22,11 @@ EF_ELIDABLE_CANNOT_RAISE = 0 #elidable function (and cannot raise) EF_LOOPINVARIANT = 1 #special: call it only once per loop EF_CANNOT_RAISE = 2 #a function which cannot raise - EF_ELIDABLE_CAN_RAISE = 3 #elidable function (but can raise) - EF_CAN_RAISE = 4 #normal function (can raise) - EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE = 5 #can raise and force virtualizables - EF_RANDOM_EFFECTS = 6 #can do whatever + EF_ELIDABLE_OR_MEMORYERROR = 3 + EF_ELIDABLE_CAN_RAISE = 4 #elidable function (but can raise) + EF_CAN_RAISE = 5 #normal function (can raise) + EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE = 6 #can raise and force virtualizables + EF_RANDOM_EFFECTS = 7 #can do whatever OS_ARRAYCOPY = 0 @@ -600,7 +601,8 @@ extraeffect = 5 EF_LOOPINVARIANT = 1 EF_ELIDABLE_CANNOT_RAISE = 2 - EF_ELIDABLE_CAN_RAISE = 3 + EF_ELIDABLE_OR_MEMORYERROR = 3 + EF_ELIDABLE_CAN_RAISE = 4 descr.get_extra_info = XTra h.invalidate_caches(rop.CALL, descr, []) assert h.is_unescaped(box1) diff --git a/rpython/jit/metainterp/test/test_string.py b/rpython/jit/metainterp/test/test_string.py --- a/rpython/jit/metainterp/test/test_string.py +++ b/rpython/jit/metainterp/test/test_string.py @@ -897,9 +897,11 @@ m -= 1 return 42 self.meta_interp(f, [6, 7]) - # xxx used to be 'call=4', but the two extra calls in the loop - # are not safe to remove; see 56eebe9dd813 - self.check_resops(unicodesetitem=2, newunicode=2, call=6, + # used to be 'call=4', but the call to ll_str2unicode in the + # loop is not safe to remove; see 56eebe9dd813. We can still + # remove the call to ll_str, because that cannot raise anything + # else than MemoryError. + self.check_resops(unicodesetitem=2, newunicode=2, call=5, copyunicodecontent=2, unicodegetitem=0) def test_str2unicode_fold(self): From noreply at buildbot.pypy.org Wed Apr 8 08:58:20 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 8 Apr 2015 08:58:20 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Close this old branch Message-ID: <20150408065820.F17241C1CD5@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1678:c9625848faed Date: 2015-04-05 10:55 +0200 http://bitbucket.org/cffi/cffi/changeset/c9625848faed/ Log: Close this old branch From noreply at buildbot.pypy.org Wed Apr 8 08:58:22 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 8 Apr 2015 08:58:22 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: A branch to really start work on cffi 1.0 Message-ID: <20150408065822.14D161C1CD5@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1679:4d5de072a07d Date: 2015-04-05 10:55 +0200 http://bitbucket.org/cffi/cffi/changeset/4d5de072a07d/ Log: A branch to really start work on cffi 1.0 From noreply at buildbot.pypy.org Wed Apr 8 08:58:23 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 8 Apr 2015 08:58:23 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Quick and dirty hack to get started Message-ID: <20150408065823.224CB1C1CD5@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1680:3fba04d45c38 Date: 2015-04-08 08:58 +0200 http://bitbucket.org/cffi/cffi/changeset/3fba04d45c38/ Log: Quick and dirty hack to get started diff --git a/cffi/api.py b/cffi/api.py --- a/cffi/api.py +++ b/cffi/api.py @@ -371,6 +371,31 @@ self._libraries.append(lib) return lib + def csource(self, modulename, source='', **kwargs): + if hasattr(self, '_csource'): + raise ValueError("can't call csource() more than once") + self._csource = (modulename, source, kwargs) + + def recompile(self): + from .verifier import Verifier + if self._windows_unicode: + self._apply_windows_unicode(kwargs) + modulename, source, kwargs = self._csource + verifier = Verifier(self, source, modulename=modulename+'_ffi', + tmpdir='.', **kwargs) + verifier.compile_module() + f = open(modulename + '.py', 'w') + print >> f, 'import os, cffi as _cffi' + print >> f, 'ffi = _cffi.FFI()' + for cdef in self._cdefsources: + print >> f + print >> f, 'ffi.cdef(%r)' % (cdef,) + print >> f + print >> f, 'lib = ffi.verify(' + print >> f, ' tmpdir=os.path.dirname(__file__), modulename=%r)' % ( + modulename + '_ffi',) + f.close() + def _get_errno(self): return self._backend.get_errno() def _set_errno(self, errno): diff --git a/demo/bsdopendirtype.py b/demo/bsdopendirtype.py --- a/demo/bsdopendirtype.py +++ b/demo/bsdopendirtype.py @@ -1,22 +1,4 @@ -from cffi import FFI - -ffi = FFI() -ffi.cdef(""" - typedef ... DIR; - struct dirent { - unsigned char d_type; /* type of file */ - char d_name[]; /* filename */ - ...; - }; - DIR *opendir(const char *name); - int closedir(DIR *dirp); - struct dirent *readdir(DIR *dirp); - static const int DT_BLK, DT_CHR, DT_DIR, DT_FIFO, DT_LNK, DT_REG, DT_SOCK; -""") -lib = ffi.verify(""" - #include - #include -""") +from _bsdopendirtype import ffi, lib def _posix_error(): @@ -60,3 +42,7 @@ yield name, smode finally: lib.closedir(dirp) + +if __name__ == '__main__': + for name, smode in opendir('/tmp'): + print hex(smode), name diff --git a/demo/bsdopendirtype_setup.py b/demo/bsdopendirtype_setup.py new file mode 100644 --- /dev/null +++ b/demo/bsdopendirtype_setup.py @@ -0,0 +1,22 @@ +from cffi import FFI + +ffi = FFI() +ffi.csource("_bsdopendirtype", """ + #include + #include +""") +ffi.cdef(""" + typedef ... DIR; + struct dirent { + unsigned char d_type; /* type of file */ + char d_name[]; /* filename */ + ...; + }; + DIR *opendir(const char *name); + int closedir(DIR *dirp); + struct dirent *readdir(DIR *dirp); + static const int DT_BLK, DT_CHR, DT_DIR, DT_FIFO, DT_LNK, DT_REG, DT_SOCK; +""") + +if __name__ == '__main__': + ffi.recompile() From noreply at buildbot.pypy.org Wed Apr 8 09:43:01 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 8 Apr 2015 09:43:01 +0200 (CEST) Subject: [pypy-commit] pypy default: Fix tests: the EF numbers have shifted Message-ID: <20150408074301.467E11C14B1@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76745:2133c4325644 Date: 2015-04-08 09:43 +0200 http://bitbucket.org/pypy/pypy/changeset/2133c4325644/ Log: Fix tests: the EF numbers have shifted diff --git a/pypy/module/pypyjit/test_pypy_c/test_buffers.py b/pypy/module/pypyjit/test_pypy_c/test_buffers.py --- a/pypy/module/pypyjit/test_pypy_c/test_buffers.py +++ b/pypy/module/pypyjit/test_pypy_c/test_buffers.py @@ -42,7 +42,7 @@ assert loop.match_by_id('unpack', """ guard_not_invalidated(descr=...) p90 = newstr(4) - call(ConstClass(copy_raw_to_string), i55, p90, 0, 4, descr=) + call(ConstClass(copy_raw_to_string), i55, p90, 0, 4, descr=) guard_no_exception(descr=...) i91 = strgetitem(p90, 0) i92 = strgetitem(p90, 1) diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py --- a/pypy/module/pypyjit/test_pypy_c/test_call.py +++ b/pypy/module/pypyjit/test_pypy_c/test_call.py @@ -355,7 +355,7 @@ i17 = arraylen_gc(p15, descr=) i18 = int_lt(i17, i15) # a cond call to _ll_list_resize_hint_really_look_inside_iff - cond_call(i18, _, p8, i15, 1, descr=) + cond_call(i18, _, p8, i15, 1, descr=) guard_no_exception(descr=...) p17 = getfield_gc(p8, descr=) setarrayitem_gc(p17, i13, i12, descr=) @@ -395,7 +395,7 @@ setarrayitem_gc(p24, 0, p26, descr=) setfield_gc(p22, p24, descr=) }}} - p32 = call_may_force(_, p18, p22, descr=) + p32 = call_may_force(_, p18, p22, descr=) ... """) diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py --- a/pypy/module/pypyjit/test_pypy_c/test_containers.py +++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py @@ -74,7 +74,7 @@ setfield_gc(p13, p15, descr=) setfield_gc(p13, ConstPtr(0), descr=) }}} - i17 = call(ConstClass(ll_dict_lookup_trampoline), p13, p10, i12, 1, descr=) + i17 = call(ConstClass(ll_dict_lookup_trampoline), p13, p10, i12, 1, descr=) {{{ setfield_gc(p13, 0, descr=) setfield_gc(p13, 0, descr=) @@ -82,10 +82,10 @@ }}} guard_no_exception(descr=...) p20 = new_with_vtable(ConstClass(W_IntObject)) - call(ConstClass(_ll_dict_setitem_lookup_done_trampoline), p13, p10, p20, i12, i17, descr=) + call(ConstClass(_ll_dict_setitem_lookup_done_trampoline), p13, p10, p20, i12, i17, descr=) setfield_gc(p20, i5, descr=) guard_no_exception(descr=...) - i23 = call(ConstClass(ll_call_lookup_function), p13, p10, i12, 0, descr=) + i23 = call(ConstClass(ll_call_lookup_function), p13, p10, i12, 0, descr=) guard_no_exception(descr=...) i27 = int_lt(i23, 0) guard_false(i27, descr=...) diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py --- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py +++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py @@ -35,7 +35,7 @@ guard_not_invalidated(descr=...) i17 = force_token() setfield_gc(p0, i17, descr=<.* .*PyFrame.vable_token .*>) - f21 = call_release_gil(%s, 2.000000, 3.000000, descr=) + f21 = call_release_gil(%s, 2.000000, 3.000000, descr=) guard_not_forced(descr=...) guard_no_exception(descr=...) """ % pow_addr) @@ -202,7 +202,7 @@ assert loop.match_by_id('cfficall', """ p96 = force_token() setfield_gc(p0, p96, descr=) - f97 = call_release_gil(91, i59, 1.0, 3, descr=) + f97 = call_release_gil(91, i59, 1.0, 3, descr=) guard_not_forced(descr=...) guard_no_exception(descr=...) """, ignore_ops=['guard_not_invalidated']) @@ -266,7 +266,7 @@ loop, = log.loops_by_id('cfficall') assert loop.match_by_id('cfficall', """ ... - f1 = call_release_gil(..., descr=) + f1 = call_release_gil(..., descr=) ... """) @@ -330,14 +330,14 @@ guard_value(p165, ConstPtr(ptr70), descr=...) p166 = getfield_gc(p165, descr=) guard_value(p166, ConstPtr(ptr72), descr=...) - p167 = call(ConstClass(_ll_0_alloc_with_del___), descr=) + p167 = call(ConstClass(_ll_0_alloc_with_del___), descr=) guard_no_exception(descr=...) i112 = int_signext(i160, 2) setfield_gc(p167, ConstPtr(ptr85), descr=) i114 = int_ne(i160, i112) guard_false(i114, descr=...) --TICK-- - i119 = call(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=) + i119 = call(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=) raw_store(i119, 0, i160, descr=) raw_store(i119, 2, i160, descr=) raw_store(i119, 4, i160, descr=) diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py --- a/pypy/module/pypyjit/test_pypy_c/test_string.py +++ b/pypy/module/pypyjit/test_pypy_c/test_string.py @@ -6,7 +6,7 @@ else: SHIFT = 63 -# XXX review the descrs to replace some EF=4 with EF=3 (elidable) +# XXX review the descrs to replace some EF=5 with EF=4 (elidable) class TestString(BaseTestPyPyC): @@ -80,12 +80,12 @@ i23 = strgetitem(p10, i19) p25 = newstr(1) strsetitem(p25, 0, i23) - p93 = call(ConstClass(fromstr), p25, 16, descr=) + p93 = call(ConstClass(fromstr), p25, 16, descr=) guard_no_exception(descr=...) i95 = getfield_gc_pure(p93, descr=) i96 = int_gt(i95, #) guard_false(i96, descr=...) - i94 = call(ConstClass(rbigint._toint_helper), p93, descr=) + i94 = call(ConstClass(rbigint._toint_helper), p93, descr=) guard_no_exception(descr=...) i95 = int_add_ovf(i6, i94) guard_no_overflow(descr=...) @@ -119,21 +119,21 @@ setfield_gc(p86, 23, descr=) setfield_gc(p86, 23, descr=) }}} - call(ConstClass(ll_append_res0__stringbuilderPtr_rpy_stringPtr), p86, p80, descr=) + call(ConstClass(ll_append_res0__stringbuilderPtr_rpy_stringPtr), p86, p80, descr=) guard_no_exception(descr=...) i89 = getfield_gc(p86, descr=) i90 = getfield_gc(p86, descr=) i91 = int_eq(i89, i90) - cond_call(i91, ConstClass(ll_grow_by__stringbuilderPtr_Signed), p86, 1, descr=) + cond_call(i91, ConstClass(ll_grow_by__stringbuilderPtr_Signed), p86, 1, descr=) guard_no_exception(descr=...) i92 = getfield_gc(p86, descr=) i93 = int_add(i92, 1) p94 = getfield_gc(p86, descr=) strsetitem(p94, i92, 32) setfield_gc(p86, i93, descr=) - call(ConstClass(ll_append_res0__stringbuilderPtr_rpy_stringPtr), p86, p80, descr=) + call(ConstClass(ll_append_res0__stringbuilderPtr_rpy_stringPtr), p86, p80, descr=) guard_no_exception(descr=...) - p95 = call(..., descr=) # ll_build + p95 = call(..., descr=) # ll_build guard_no_exception(descr=...) i96 = strlen(p95) i97 = int_add_ovf(i71, i96) @@ -248,7 +248,7 @@ i50 = int_add(i47, 1) setfield_gc(p15, i50, descr=) guard_not_invalidated(descr=...) - p52 = call(ConstClass(str_decode_ascii__raise_unicode_exception_decode), ConstPtr(ptr38), 3, 1, descr=) + p52 = call(ConstClass(str_decode_ascii__raise_unicode_exception_decode), ConstPtr(ptr38), 3, 1, descr=) guard_no_exception(descr=...) p53 = getfield_gc_pure(p52, descr=) guard_nonnull(p53, descr=...) diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py --- a/pypy/module/pypyjit/test_pypy_c/test_thread.py +++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py @@ -64,7 +64,7 @@ guard_true(i56, descr=...) p57 = force_token() setfield_gc(p0, p57, descr=) - i58 = call_release_gil(0, _, i37, 1, descr=) + i58 = call_release_gil(0, _, i37, 1, descr=) guard_not_forced(descr=...) guard_no_exception(descr=...) i58 = int_sub(i44, 1) diff --git a/pypy/module/pypyjit/test_pypy_c/test_weakref.py b/pypy/module/pypyjit/test_pypy_c/test_weakref.py --- a/pypy/module/pypyjit/test_pypy_c/test_weakref.py +++ b/pypy/module/pypyjit/test_pypy_c/test_weakref.py @@ -35,7 +35,7 @@ guard_nonnull_class(p66, ..., descr=...) p67 = force_token() setfield_gc(p0, p67, descr=) - p68 = call_may_force(ConstClass(WeakrefLifelineWithCallbacks.make_weakref_with_callback), p66, ConstPtr(ptr50), p14, ConstPtr(ptr51), descr=) + p68 = call_may_force(ConstClass(WeakrefLifelineWithCallbacks.make_weakref_with_callback), p66, ConstPtr(ptr50), p14, ConstPtr(ptr51), descr=) guard_not_forced(descr=...) guard_no_exception(descr=...) guard_nonnull_class(p68, ..., descr=...) From noreply at buildbot.pypy.org Wed Apr 8 09:48:13 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 8 Apr 2015 09:48:13 +0200 (CEST) Subject: [pypy-commit] pypy default: Add the guard_no_exception() that should be there Message-ID: <20150408074813.D8AD91C00FA@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76746:fa50a901f61b Date: 2015-04-08 09:48 +0200 http://bitbucket.org/pypy/pypy/changeset/fa50a901f61b/ Log: Add the guard_no_exception() that should be there diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -3633,6 +3633,7 @@ [p1, i1, i4] setfield_gc(p1, i1, descr=valuedescr) i3 = call_pure(p1, descr=elidable2calldescr) + guard_no_exception() [] setfield_gc(p1, i3, descr=valuedescr) jump(p1, i4, i3) ''' @@ -3645,6 +3646,7 @@ [p1, i1, i4] setfield_gc(p1, i1, descr=valuedescr) i3 = call(p1, descr=elidable2calldescr) + guard_no_exception() [] setfield_gc(p1, i3, descr=valuedescr) i148 = same_as(i3) i147 = same_as(i3) @@ -3662,6 +3664,7 @@ [p1, i1, i4] setfield_gc(p1, i1, descr=valuedescr) i3 = call_pure(p1, descr=elidable3calldescr) + guard_no_exception() [] setfield_gc(p1, i3, descr=valuedescr) jump(p1, i4, i3) ''' @@ -3669,6 +3672,7 @@ [p1, i1, i4] setfield_gc(p1, i1, descr=valuedescr) i3 = call(p1, descr=elidable3calldescr) + guard_no_exception() [] setfield_gc(p1, i3, descr=valuedescr) jump(p1, i4, i3) ''' @@ -3676,6 +3680,7 @@ [p1, i1, i4] setfield_gc(p1, i1, descr=valuedescr) i3 = call(p1, descr=elidable3calldescr) + guard_no_exception() [] setfield_gc(p1, i3, descr=valuedescr) i167 = same_as(i3) jump(p1, i4, i3) From noreply at buildbot.pypy.org Wed Apr 8 10:00:08 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 8 Apr 2015 10:00:08 +0200 (CEST) Subject: [pypy-commit] pypy default: Add a passing test Message-ID: <20150408080008.5F0D81C14BF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76747:46adf3dd3cb9 Date: 2015-04-08 09:56 +0200 http://bitbucket.org/pypy/pypy/changeset/46adf3dd3cb9/ Log: Add a passing test diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -3737,6 +3737,29 @@ ''' self.optimize_loop(ops, expected, preamble, call_pure_results) + def test_call_pure_constant_folding_memoryerr(self): + ops = ''' + [p0, i0] + escape(i0) + i3 = call_pure(123456, p0, descr=elidable2calldescr) + guard_no_exception() [] + jump(p0, i3) + ''' + preamble = ''' + [p0, i0] + escape(i0) + i3 = call(123456, p0, descr=elidable2calldescr) + guard_no_exception() [] + i4 = same_as(i3) + jump(p0, i3, i4) + ''' + expected = ''' + [p0, i3, i4] + escape(i3) + jump(p0, i4, i4) + ''' + self.optimize_loop(ops, expected, preamble) + def test_call_pure_constant_folding_exc(self): # CALL_PURE may be followed by GUARD_NO_EXCEPTION # we can't remove such call_pures from the loop, From noreply at buildbot.pypy.org Wed Apr 8 10:00:09 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 8 Apr 2015 10:00:09 +0200 (CEST) Subject: [pypy-commit] pypy default: Test and fix Message-ID: <20150408080009.ADE901C14BF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76748:523b8af1cd1c Date: 2015-04-08 10:00 +0200 http://bitbucket.org/pypy/pypy/changeset/523b8af1cd1c/ Log: Test and fix diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -3807,6 +3807,7 @@ ops = ''' [p1, i1, i2] p2 = call_pure(0, p1, i1, i2, descr=strslicedescr) + guard_no_exception() [] escape(p2) jump(p1, i1, i2) ''' diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py --- a/rpython/jit/metainterp/optimizeopt/vstring.py +++ b/rpython/jit/metainterp/optimizeopt/vstring.py @@ -597,6 +597,7 @@ vright.ensure_nonnull() value = self.make_vstring_concat(op.result, op, mode) value.setup(vleft, vright) + self.last_emitted_operation = REMOVED return True def opt_call_stroruni_STR_SLICE(self, op, mode): @@ -626,6 +627,7 @@ # value = self.make_vstring_slice(op.result, op, mode) value.setup(vstr, vstart, self.getvalue(lengthbox)) + self.last_emitted_operation = REMOVED return True def opt_call_stroruni_STR_EQUAL(self, op, mode): From noreply at buildbot.pypy.org Wed Apr 8 11:11:32 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 8 Apr 2015 11:11:32 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: fix Message-ID: <20150408091132.4CB851C1151@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1681:f56ddf6bed4a Date: 2015-04-08 11:12 +0200 http://bitbucket.org/cffi/cffi/changeset/f56ddf6bed4a/ Log: fix diff --git a/cffi/api.py b/cffi/api.py --- a/cffi/api.py +++ b/cffi/api.py @@ -378,9 +378,9 @@ def recompile(self): from .verifier import Verifier + modulename, source, kwargs = self._csource if self._windows_unicode: self._apply_windows_unicode(kwargs) - modulename, source, kwargs = self._csource verifier = Verifier(self, source, modulename=modulename+'_ffi', tmpdir='.', **kwargs) verifier.compile_module() From noreply at buildbot.pypy.org Wed Apr 8 11:39:22 2015 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 8 Apr 2015 11:39:22 +0200 (CEST) Subject: [pypy-commit] pypy default: fix for cpython issue #23881 - replace test ftp servers, also close connections Message-ID: <20150408093922.6C5E51C00FA@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76749:e73c45af4c52 Date: 2015-04-08 10:40 +0300 http://bitbucket.org/pypy/pypy/changeset/e73c45af4c52/ Log: fix for cpython issue #23881 - replace test ftp servers, also close connections diff --git a/lib-python/2.7/test/test_urllib2net.py b/lib-python/2.7/test/test_urllib2net.py --- a/lib-python/2.7/test/test_urllib2net.py +++ b/lib-python/2.7/test/test_urllib2net.py @@ -102,11 +102,8 @@ def test_ftp(self): urls = [ - 'ftp://ftp.kernel.org/pub/linux/kernel/README', - 'ftp://ftp.kernel.org/pub/linux/kernel/non-existent-file', - #'ftp://ftp.kernel.org/pub/leenox/kernel/test', - 'ftp://gatekeeper.research.compaq.com/pub/DEC/SRC' - '/research-reports/00README-Legal-Rules-Regs', + 'ftp://ftp.debian.org/debian/README', + 'ftp://ftp.debian.org/debian/non-existent-file', ] self._test_urls(urls, self._extra_handlers()) @@ -255,6 +252,7 @@ with test_support.transient_internet(url, timeout=None): u = _urlopen_with_retry(url) self.assertIsNone(u.fp._sock.fp._sock.gettimeout()) + u.close() def test_http_default_timeout(self): self.assertIsNone(socket.getdefaulttimeout()) @@ -266,6 +264,7 @@ finally: socket.setdefaulttimeout(None) self.assertEqual(u.fp._sock.fp._sock.gettimeout(), 60) + u.close() def test_http_no_timeout(self): self.assertIsNone(socket.getdefaulttimeout()) @@ -277,20 +276,23 @@ finally: socket.setdefaulttimeout(None) self.assertIsNone(u.fp._sock.fp._sock.gettimeout()) + u.close() def test_http_timeout(self): url = "http://www.example.com" with test_support.transient_internet(url): u = _urlopen_with_retry(url, timeout=120) self.assertEqual(u.fp._sock.fp._sock.gettimeout(), 120) + u.close() - FTP_HOST = "ftp://ftp.mirror.nl/pub/gnu/" + FTP_HOST = 'ftp://ftp.debian.org/debian/' def test_ftp_basic(self): self.assertIsNone(socket.getdefaulttimeout()) with test_support.transient_internet(self.FTP_HOST, timeout=None): u = _urlopen_with_retry(self.FTP_HOST) self.assertIsNone(u.fp.fp._sock.gettimeout()) + u.close() def test_ftp_default_timeout(self): self.assertIsNone(socket.getdefaulttimeout()) @@ -301,6 +303,7 @@ finally: socket.setdefaulttimeout(None) self.assertEqual(u.fp.fp._sock.gettimeout(), 60) + u.close() def test_ftp_no_timeout(self): self.assertIsNone(socket.getdefaulttimeout(),) @@ -311,11 +314,16 @@ finally: socket.setdefaulttimeout(None) self.assertIsNone(u.fp.fp._sock.gettimeout()) + u.close() def test_ftp_timeout(self): with test_support.transient_internet(self.FTP_HOST): - u = _urlopen_with_retry(self.FTP_HOST, timeout=60) + try: + u = _urlopen_with_retry(self.FTP_HOST, timeout=60) + except: + raise self.assertEqual(u.fp.fp._sock.gettimeout(), 60) + u.close() def test_main(): From noreply at buildbot.pypy.org Wed Apr 8 11:49:42 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 8 Apr 2015 11:49:42 +0200 (CEST) Subject: [pypy-commit] benchmarks single-run: merge default Message-ID: <20150408094942.518B71C0EEE@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: single-run Changeset: r321:15d8a7e08f61 Date: 2015-04-08 11:12 +0200 http://bitbucket.org/pypy/benchmarks/changeset/15d8a7e08f61/ Log: merge default diff too long, truncating to 2000 out of 144423 lines diff --git a/benchmarks.py b/benchmarks.py --- a/benchmarks.py +++ b/benchmarks.py @@ -61,7 +61,9 @@ opts = { 'eparse' : {'bm_env': {'PYTHONPATH': relative('lib/monte')}}, 'bm_mako' : {'bm_env': {'PYTHONPATH': relative('lib/mako')}}, - 'bm_chameleon': {'bm_env': {'PYTHONPATH': relative('lib/chameleon/src')}}, + 'bm_dulwich_log': {'bm_env': {'PYTHONPATH': relative('lib/dulwich-0.9.1')}}, + 'bm_chameleon': {'bm_env': {'PYTHONPATH': relative('lib/chameleon/src')}, + 'iteration_scaling': 3}, } for name in ['expand', 'integrate', 'sum', 'str']: @@ -77,12 +79,14 @@ for name in ['float', 'nbody_modified', 'meteor-contest', 'fannkuch', 'spectral-norm', 'chaos', 'telco', 'go', 'pyflate-fast', 'raytrace-simple', 'crypto_pyaes', 'bm_mako', 'bm_chameleon', - 'json_bench', 'pidigits', 'hexiom2', 'eparse']: + 'json_bench', 'pidigits', 'hexiom2', 'eparse', 'deltablue', + 'bm_dulwich_log', 'bm_krakatau', 'bm_mdp', 'pypy_interp', + 'bm_icbd']: _register_new_bm(name, name, globals(), **opts.get(name, {})) for name in ['names', 'iteration', 'tcp', 'pb', ]:#'web']:#, 'accepts']: _register_new_bm_twisted(name, 'twisted_' + name, - globals(), bm_env={'PYTHONPATH': ':'.join(TWISTED)}) + globals(), bm_env={'PYTHONPATH': os.pathsep.join(TWISTED)}) _register_new_bm('spitfire', 'spitfire', globals(), extra_args=['--benchmark=spitfire_o4']) diff --git a/io/iobasic.py b/io/iobasic.py new file mode 100644 --- /dev/null +++ b/io/iobasic.py @@ -0,0 +1,43 @@ + +import os, time + +def measure(func, num1, num2): + t0 = time.time() + func(num1, num2) + print func.__code__.co_name, "%d bytes, %.2fus per write" % ( + num2, (time.time() - t0) / num1 * 1000 * 1000) + +def fwrite(num, num2): + fd = os.open("/dev/null", os.O_WRONLY) + for i in range(num): + os.write(fd, " " * num2) + +def fread(num, num2): + fd = os.open("/dev/full", os.O_RDONLY) + for i in range(num): + os.read(fd, num2) + +def file_write(num, num2): + f = open("/dev/null", "w") + for i in range(num): + f.write(" " * num2) + f.flush() + +def file_read(num, num2): + f = open("/dev/full") + for i in range(num): + f.read(num2) + +if __name__ == '__main__': + measure(fread, 10000000, 100) + measure(fwrite, 10000000, 100) + measure(fread, 10000000, 1000) + measure(fwrite, 10000000, 1000) + measure(fread, 1000000, 10000) + measure(fwrite, 1000000, 10000) + measure(file_read, 10000000, 100) + measure(file_write, 10000000, 100) + measure(file_read, 10000000, 1000) + measure(file_write, 10000000, 1000) + measure(file_read, 1000000, 10000) + measure(file_write, 1000000, 10000) diff --git a/lib/dulwich-0.9.1/COPYING b/lib/dulwich-0.9.1/COPYING new file mode 100644 --- /dev/null +++ b/lib/dulwich-0.9.1/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/lib/dulwich-0.9.1/MANIFEST.in b/lib/dulwich-0.9.1/MANIFEST.in new file mode 100644 --- /dev/null +++ b/lib/dulwich-0.9.1/MANIFEST.in @@ -0,0 +1,7 @@ +include NEWS +include README +include Makefile +include COPYING +recursive-include docs conf.py *.txt Makefile make.bat +recursive-include examples *.py +graft dulwich/tests/data diff --git a/lib/dulwich-0.9.1/Makefile b/lib/dulwich-0.9.1/Makefile new file mode 100644 --- /dev/null +++ b/lib/dulwich-0.9.1/Makefile @@ -0,0 +1,49 @@ +PYTHON = python +SETUP = $(PYTHON) setup.py +PYDOCTOR ?= pydoctor +ifeq ($(shell $(PYTHON) -c "import sys; print sys.version_info >= (2, 7)"),True) +TESTRUNNER ?= unittest +else +TESTRUNNER ?= unittest2.__main__ +endif +RUNTEST = PYTHONPATH=.:$(PYTHONPATH) $(PYTHON) -m $(TESTRUNNER) + +DESTDIR=/ + +all: build + +doc:: pydoctor + +pydoctor:: + $(PYDOCTOR) --make-html -c dulwich.cfg + +build:: + $(SETUP) build + $(SETUP) build_ext -i + +install:: + $(SETUP) install --root="$(DESTDIR)" + +check:: build + $(RUNTEST) dulwich.tests.test_suite + +check-tutorial:: build + $(RUNTEST) dulwich.tests.tutorial_test_suite + +check-nocompat:: build + $(RUNTEST) dulwich.tests.nocompat_test_suite + +check-compat:: build + $(RUNTEST) dulwich.tests.compat_test_suite + +check-pypy:: clean + $(MAKE) check-noextensions PYTHON=pypy + +check-noextensions:: clean + $(RUNTEST) dulwich.tests.test_suite + +check-all: check check-pypy check-noextensions + +clean:: + $(SETUP) clean --all + rm -f dulwich/*.so diff --git a/lib/dulwich-0.9.1/NEWS b/lib/dulwich-0.9.1/NEWS new file mode 100644 --- /dev/null +++ b/lib/dulwich-0.9.1/NEWS @@ -0,0 +1,871 @@ +0.9.1 2013-09-22 + + BUG FIXES + + * Support lookups of 40-character refs in BaseRepo.__getitem__. (Chow Loong Jin, Jelmer Vernooij) + + * Fix fetching packs with side-band-64k capability disabled. (David Keijser, Jelmer Vernooij) + + * Several fixes in send-pack protocol behaviour - handling of empty pack files and deletes. + (milki, #1063087) + + * Fix capability negotiation when fetching packs over HTTP. + (#1072461, William Grant) + + * Enforce determine_wants returning an empty list rather than None. (Fabien Boucher, Jelmer Vernooij) + + * In the server, support pushes just removing refs. (Fabien Boucher, Jelmer Vernooij) + + IMPROVEMENTS + + * Support passing a single revision to BaseRepo.get_walker() rather than a list of revisions. + (Alberto Ruiz) + + * Add `Repo.get_description` method. (Jelmer Vernooij) + + * Support thin packs in Pack.iterobjects() and Pack.get_raw(). + (William Grant) + + * Add `MemoryObjectStore.add_pack` and `MemoryObjectStore.add_thin_pack` methods. + (David Bennett) + + * Add paramiko-based SSH vendor. (Aaron O'Mullan) + + * Support running 'dulwich.server' and 'dulwich.web' using 'python -m'. + (Jelmer Vernooij) + + * Add ObjectStore.close(). (Jelmer Vernooij) + + * Raise appropriate NotImplementedError when encountering dumb HTTP servers. + (Jelmer Vernooij) + + API CHANGES + + * SSHVendor.connect_ssh has been renamed to SSHVendor.run_command. + (Jelmer Vernooij) + + * ObjectStore.add_pack() now returns a 3-tuple. The last element will be an + abort() method that can be used to cancel the pack operation. + (Jelmer Vernooij) + +0.9.0 2013-05-31 + + BUG FIXES + + * Push efficiency - report missing objects only. (#562676, Artem Tikhomirov) + + * Use indentation consistent with C Git in config files. + (#1031356, Curt Moore, Jelmer Vernooij) + + * Recognize and skip binary files in diff function. + (Takeshi Kanemoto) + + * Fix handling of relative paths in dulwich.client.get_transport_and_path. + (Brian Visel, #1169368) + + * Preserve ordering of entries in configuration. + (Benjamin Pollack) + + * Support ~ expansion in SSH client paths. (milki, #1083439) + + * Support relative paths in alternate paths. + (milki, Michel Lespinasse, #1175007) + + * Log all error messages from wsgiref server to the logging module. This + makes the test suit quiet again. (Gary van der Merwe) + + * Support passing None for empty tree in changes_from_tree. + (Kevin Watters) + + * Support fetching empty repository in client. (milki, #1060462) + + IMPROVEMENTS: + + * Add optional honor_filemode flag to build_index_from_tree. + (Mark Mikofski) + + * Support core/filemode setting when building trees. (Jelmer Vernooij) + + * Add chapter on tags in tutorial. (Ryan Faulkner) + + FEATURES + + * Add support for mergetags. (milki, #963525) + + * Add support for posix shell hooks. (milki) + +0.8.7 2012-11-27 + + BUG FIXES + + * Fix use of alternates in ``DiskObjectStore``.{__contains__,__iter__}. + (Dmitriy) + + * Fix compatibility with Python 2.4. (David Carr) + +0.8.6 2012-11-09 + + API CHANGES + + * dulwich.__init__ no longer imports client, protocol, repo and + server modules. (Jelmer Vernooij) + + FEATURES + + * ConfigDict now behaves more like a dictionary. + (Adam 'Cezar' Jenkins, issue #58) + + * HTTPGitApplication now takes an optional + `fallback_app` argument. (Jonas Haag, issue #67) + + * Support for large pack index files. (Jameson Nash) + + TESTING + + * Make index entry tests a little bit less strict, to cope with + slightly different behaviour on various platforms. + (Jelmer Vernooij) + + * ``setup.py test`` (available when setuptools is installed) now + runs all tests, not just the basic unit tests. + (Jelmer Vernooij) + + BUG FIXES + + * Commit._deserialize now actually deserializes the current state rather than + the previous one. (Yifan Zhang, issue #59) + + * Handle None elements in lists of TreeChange objects. (Alex Holmes) + + * Support cloning repositories without HEAD set. + (D-Key, Jelmer Vernooij, issue #69) + + * Support ``MemoryRepo.get_config``. (Jelmer Vernooij) + + * In ``get_transport_and_path``, pass extra keyword arguments on to + HttpGitClient. (Jelmer Vernooij) + +0.8.5 2012-03-29 + + BUG FIXES + + * Avoid use of 'with' in dulwich.index. (Jelmer Vernooij) + + * Be a little bit strict about OS behaviour in index tests. + Should fix the tests on Debian GNU/kFreeBSD. (Jelmer Vernooij) + +0.8.4 2012-03-28 + + BUG FIXES + + * Options on the same line as sections in config files are now supported. + (Jelmer Vernooij, #920553) + + * Only negotiate capabilities that are also supported by the server. + (Rod Cloutier, Risto Kankkunen) + + * Fix parsing of invalid timezone offsets with two minus signs. + (Jason R. Coombs, #697828) + + * Reset environment variables during tests, to avoid + test isolation leaks reading ~/.gitconfig. (Risto Kankkunen) + + TESTS + + * $HOME is now explicitly specified for tests that use it to read + ``~/.gitconfig``, to prevent test isolation issues. + (Jelmer Vernooij, #920330) + + FEATURES + + * Additional arguments to get_transport_and_path are now passed + on to the constructor of the transport. (Sam Vilain) + + * The WSGI server now transparently handles when a git client submits data + using Content-Encoding: gzip. + (David Blewett, Jelmer Vernooij) + + * Add dulwich.index.build_index_from_tree(). (milki) + +0.8.3 2012-01-21 + + FEATURES + + * The config parser now supports the git-config file format as + described in git-config(1) and can write git config files. + (Jelmer Vernooij, #531092, #768687) + + * ``Repo.do_commit`` will now use the user identity from + .git/config or ~/.gitconfig if none was explicitly specified. + (Jelmer Vernooij) + + BUG FIXES + + * Allow ``determine_wants`` methods to include the zero sha in their + return value. (Jelmer Vernooij) + +0.8.2 2011-12-18 + + BUG FIXES + + * Cope with different zlib buffer sizes in sha1 file parser. + (Jelmer Vernooij) + + * Fix get_transport_and_path for HTTP/HTTPS URLs. + (Bruno Renié) + + * Avoid calling free_objects() on NULL in error cases. (Chris Eberle) + + * Fix use --bare argument to 'dulwich init'. (Chris Eberle) + + * Properly abort connections when the determine_wants function + raises an exception. (Jelmer Vernooij, #856769) + + * Tweak xcodebuild hack to deal with more error output. + (Jelmer Vernooij, #903840) + + FEATURES + + * Add support for retrieving tarballs from remote servers. + (Jelmer Vernooij, #379087) + + * New method ``update_server_info`` which generates data + for dumb server access. (Jelmer Vernooij, #731235) + +0.8.1 2011-10-31 + + FEATURES + + * Repo.do_commit has a new argument 'ref'. + + * Repo.do_commit has a new argument 'merge_heads'. (Jelmer Vernooij) + + * New ``Repo.get_walker`` method. (Jelmer Vernooij) + + * New ``Repo.clone`` method. (Jelmer Vernooij, #725369) + + * ``GitClient.send_pack`` now supports the 'side-band-64k' capability. + (Jelmer Vernooij) + + * ``HttpGitClient`` which supports the smart server protocol over + HTTP. "dumb" access is not yet supported. (Jelmer Vernooij, #373688) + + * Add basic support for alternates. (Jelmer Vernooij, #810429) + + CHANGES + + * unittest2 or python >= 2.7 is now required for the testsuite. + testtools is no longer supported. (Jelmer Vernooij, #830713) + + BUG FIXES + + * Fix compilation with older versions of MSVC. (Martin gz) + + * Special case 'refs/stash' as a valid ref. (Jelmer Vernooij, #695577) + + * Smart protocol clients can now change refs even if they are + not uploading new data. (Jelmer Vernooij, #855993) + + * Don't compile C extensions when running in pypy. + (Ronny Pfannschmidt, #881546) + + * Use different name for strnlen replacement function to avoid clashing + with system strnlen. (Jelmer Vernooij, #880362) + + API CHANGES + + * ``Repo.revision_history`` is now deprecated in favor of ``Repo.get_walker``. + (Jelmer Vernooij) + +0.8.0 2011-08-07 + + FEATURES + + * New DeltaChainIterator abstract class for quickly iterating all objects in + a pack, with implementations for pack indexing and inflation. + (Dave Borowitz) + + * New walk module with a Walker class for customizable commit walking. + (Dave Borowitz) + + * New tree_changes_for_merge function in diff_tree. (Dave Borowitz) + + * Easy rename detection in RenameDetector even without find_copies_harder. + (Dave Borowitz) + + BUG FIXES + + * Avoid storing all objects in memory when writing pack. + (Jelmer Vernooij, #813268) + + * Support IPv6 for git:// connections. (Jelmer Vernooij, #801543) + + * Improve performance of Repo.revision_history(). (Timo Schmid, #535118) + + * Fix use of SubprocessWrapper on Windows. (Paulo Madeira, #670035) + + * Fix compilation on newer versions of Mac OS X (Lion and up). (Ryan McKern, #794543) + + * Prevent raising ValueError for correct refs in RefContainer.__delitem__. + + * Correctly return a tuple from MemoryObjectStore.get_raw. (Dave Borowitz) + + * Fix a bug in reading the pack checksum when there are fewer than 20 bytes + left in the buffer. (Dave Borowitz) + + * Support ~ in git:// URL paths. (Jelmer Vernooij, #813555) + + * Make ShaFile.__eq__ work when other is not a ShaFile. (Dave Borowitz) + + * ObjectStore.get_graph_walker() now no longer yields the same + revision more than once. This has a significant improvement for + performance when wide revision graphs are involved. + (Jelmer Vernooij, #818168) + + * Teach ReceivePackHandler how to read empty packs. (Dave Borowitz) + + * Don't send a pack with duplicates of the same object. (Dave Borowitz) + + * Teach the server how to serve a clone of an empty repo. (Dave Borowitz) + + * Correctly advertise capabilities during receive-pack. (Dave Borowitz) + + * Fix add/add and add/rename conflicts in tree_changes_for_merge. + (Dave Borowitz) + + * Use correct MIME types in web server. (Dave Borowitz) + + API CHANGES + + * write_pack no longer takes the num_objects argument and requires an object + to be passed in that is iterable (rather than an iterator) and that + provides __len__. (Jelmer Vernooij) + + * write_pack_data has been renamed to write_pack_objects and no longer takes a + num_objects argument. (Jelmer Vernooij) + + * take_msb_bytes, read_zlib_chunks, unpack_objects, and + PackStreamReader.read_objects now take an additional argument indicating a + crc32 to compute. (Dave Borowitz) + + * PackObjectIterator was removed; its functionality is still exposed by + PackData.iterobjects. (Dave Borowitz) + + * Add a sha arg to write_pack_object to incrementally compute a SHA. + (Dave Borowitz) + + * Include offset in PackStreamReader results. (Dave Borowitz) + + * Move PackStreamReader from server to pack. (Dave Borowitz) + + * Extract a check_length_and_checksum, compute_file_sha, and + pack_object_header pack helper functions. (Dave Borowitz) + + * Extract a compute_file_sha function. (Dave Borowitz) + + * Remove move_in_thin_pack as a separate method; add_thin_pack now completes + the thin pack and moves it in in one step. Remove ThinPackData as well. + (Dave Borowitz) + + * Custom buffer size in read_zlib_chunks. (Dave Borowitz) + + * New UnpackedObject data class that replaces ad-hoc tuples in the return + value of unpack_object and various DeltaChainIterator methods. + (Dave Borowitz) + + * Add a lookup_path convenience method to Tree. (Dave Borowitz) + + * Optionally create RenameDetectors without passing in tree SHAs. + (Dave Borowitz) + + * Optionally include unchanged entries in RenameDetectors. (Dave Borowitz) + + * Optionally pass a RenameDetector to tree_changes. (Dave Borowitz) + + * Optionally pass a request object through to server handlers. (Dave Borowitz) + + TEST CHANGES + + * If setuptools is installed, "python setup.py test" will now run the testsuite. + (Jelmer Vernooij) + + * Add a new build_pack test utility for building packs from a simple spec. + (Dave Borowitz) + + * Add a new build_commit_graph test utility for building commits from a + simple spec. (Dave Borowitz) + +0.7.1 2011-04-12 + + BUG FIXES + + * Fix double decref in _diff_tree.c. (Ted Horst, #715528) + + * Fix the build on Windows. (Pascal Quantin) + + * Fix get_transport_and_path compatibility with pre-2.6.5 versions of Python. + (Max Bowsher, #707438) + + * BaseObjectStore.determine_wants_all no longer breaks on zero SHAs. + (Jelmer Vernooij) + + * write_tree_diff() now supports submodules. + (Jelmer Vernooij) + + * Fix compilation for XCode 4 and older versions of distutils.sysconfig. + (Daniele Sluijters) + + IMPROVEMENTS + + * Sphinxified documentation. (Lukasz Balcerzak) + + * Add Pack.keep.(Marc Brinkmann) + + API CHANGES + + * The order of the parameters to Tree.add(name, mode, sha) has changed, and + is now consistent with the rest of Dulwich. Existing code will still + work but print a DeprecationWarning. (Jelmer Vernooij, #663550) + + * Tree.entries() is now deprecated in favour of Tree.items() and + Tree.iteritems(). (Jelmer Vernooij) + +0.7.0 2011-01-21 + + FEATURES + + * New `dulwich.diff_tree` module for simple content-based rename detection. + (Dave Borowitz) + + * Add Tree.items(). (Jelmer Vernooij) + + * Add eof() and unread_pkt_line() methods to Protocol. (Dave Borowitz) + + * Add write_tree_diff(). (Jelmer Vernooij) + + * Add `serve_command` function for git server commands as executables. + (Jelmer Vernooij) + + * dulwich.client.get_transport_and_path now supports rsync-style repository URLs. + (Dave Borowitz, #568493) + + BUG FIXES + + * Correct short-circuiting operation for no-op fetches in the server. + (Dave Borowitz) + + * Support parsing git mbox patches without a version tail, as generated by + Mercurial. (Jelmer Vernooij) + + * Fix dul-receive-pack and dul-upload-pack. (Jelmer Vernooij) + + * Zero-padded file modes in Tree objects no longer trigger an exception but + the check code warns about them. (Augie Fackler, #581064) + + * Repo.init() now honors the mkdir flag. (#671159) + + * The ref format is now checked when setting a ref rather than when reading it back. + (Dave Borowitz, #653527) + + * Make sure pack files are closed correctly. (Tay Ray Chuan) + + DOCUMENTATION + + * Run the tutorial inside the test suite. (Jelmer Vernooij) + + * Reorganized and updated the tutorial. (Jelmer Vernooij, Dave Borowitz, #610550, + #610540) + + +0.6.2 2010-10-16 + + BUG FIXES + + * HTTP server correctly handles empty CONTENT_LENGTH. (Dave Borowitz) + + * Don't error when creating GitFiles with the default mode. (Dave Borowitz) + + * ThinPackData.from_file now works with resolve_ext_ref callback. + (Dave Borowitz) + + * Provide strnlen() on mingw32 which doesn't have it. (Hans Kolek) + + * Set bare=true in the configuratin for bare repositories. (Dirk Neumann) + + FEATURES + + * Use slots for core objects to save up on memory. (Jelmer Vernooij) + + * Web server supports streaming progress/pack output. (Dave Borowitz) + + * New public function dulwich.pack.write_pack_header. (Dave Borowitz) + + * Distinguish between missing files and read errors in HTTP server. + (Dave Borowitz) + + * Initial work on support for fastimport using python-fastimport. + (Jelmer Vernooij) + + * New dulwich.pack.MemoryPackIndex class. (Jelmer Vernooij) + + * Delegate SHA peeling to the object store. (Dave Borowitz) + + TESTS + + * Use GitFile when modifying packed-refs in tests. (Dave Borowitz) + + * New tests in test_web with better coverage and fewer ad-hoc mocks. + (Dave Borowitz) + + * Standardize quote delimiters in test_protocol. (Dave Borowitz) + + * Fix use when testtools is installed. (Jelmer Vernooij) + + * Add trivial test for write_pack_header. (Jelmer Vernooij) + + * Refactor some of dulwich.tests.compat.server_utils. (Dave Borowitz) + + * Allow overwriting id property of objects in test utils. (Dave Borowitz) + + * Use real in-memory objects rather than stubs for server tests. + (Dave Borowitz) + + * Clean up MissingObjectFinder. (Dave Borowitz) + + API CHANGES + + * ObjectStore.iter_tree_contents now walks contents in depth-first, sorted + order. (Dave Borowitz) + + * ObjectStore.iter_tree_contents can optionally yield tree objects as well. + (Dave Borowitz). + + * Add side-band-64k support to ReceivePackHandler. (Dave Borowitz) + + * Change server capabilities methods to classmethods. (Dave Borowitz) + + * Tweak server handler injection. (Dave Borowitz) + + * PackIndex1 and PackIndex2 now subclass FilePackIndex, which is + itself a subclass of PackIndex. (Jelmer Vernooij) + + DOCUMENTATION + + * Add docstrings for various functions in dulwich.objects. (Jelmer Vernooij) + + * Clean up docstrings in dulwich.protocol. (Dave Borowitz) + + * Explicitly specify allowed protocol commands to + ProtocolGraphWalker.read_proto_line. (Dave Borowitz) + + * Add utility functions to DictRefsContainer. (Dave Borowitz) + + +0.6.1 2010-07-22 + + BUG FIXES + + * Fix memory leak in C implementation of sorted_tree_items. (Dave Borowitz) + + * Use correct path separators for named repo files. (Dave Borowitz) + + * python > 2.7 and testtools-based test runners will now also pick up skipped + tests correctly. (Jelmer Vernooij) + + FEATURES + + * Move named file initilization to BaseRepo. (Dave Borowitz) + + * Add logging utilities and git/HTTP server logging. (Dave Borowitz) + + * The GitClient interface has been cleaned up and instances are now reusable. + (Augie Fackler) + + * Allow overriding paths to executables in GitSSHClient. + (Ross Light, Jelmer Vernooij, #585204) + + * Add PackBasedObjectStore.pack_loose_objects(). (Jelmer Vernooij) + + TESTS + + * Add tests for sorted_tree_items and C implementation. (Dave Borowitz) + + * Add a MemoryRepo that stores everything in memory. (Dave Borowitz) + + * Quiet logging output from web tests. (Dave Borowitz) + + * More flexible version checking for compat tests. (Dave Borowitz) + + * Compat tests for servers with and without side-band-64k. (Dave Borowitz) + + CLEANUP + + * Clean up file headers. (Dave Borowitz) + + TESTS + + * Use GitFile when modifying packed-refs in tests. (Dave Borowitz) + + API CHANGES + + * dulwich.pack.write_pack_index_v{1,2} now take a file-like object + rather than a filename. (Jelmer Vernooij) + + * Make dul-daemon/dul-web trivial wrappers around server functionality. + (Dave Borowitz) + + * Move reference WSGI handler to web.py. (Dave Borowitz) + + * Factor out _report_status in ReceivePackHandler. (Dave Borowitz) + + * Factor out a function to convert a line to a pkt-line. (Dave Borowitz) + + +0.6.0 2010-05-22 + +note: This list is most likely incomplete for 0.6.0. + + BUG FIXES + + * Fix ReceivePackHandler to disallow removing refs without delete-refs. + (Dave Borowitz) + + * Deal with capabilities required by the client, even if they + can not be disabled in the server. (Dave Borowitz) + + * Fix trailing newlines in generated patch files. + (Jelmer Vernooij) + + * Implement RefsContainer.__contains__. (Jelmer Vernooij) + + * Cope with \r in ref files on Windows. ( + http://github.com/jelmer/dulwich/issues/#issue/13, Jelmer Vernooij) + + * Fix GitFile breakage on Windows. (Anatoly Techtonik, #557585) + + * Support packed ref deletion with no peeled refs. (Augie Fackler) + + * Fix send pack when there is nothing to fetch. (Augie Fackler) + + * Fix fetch if no progress function is specified. (Augie Fackler) + + * Allow double-staging of files that are deleted in the index. + (Dave Borowitz) + + * Fix RefsContainer.add_if_new to support dangling symrefs. + (Dave Borowitz) + + * Non-existant index files in non-bare repositories are now treated as + empty. (Dave Borowitz) + + * Always update ShaFile.id when the contents of the object get changed. + (Jelmer Vernooij) + + * Various Python2.4-compatibility fixes. (Dave Borowitz) + + * Fix thin pack handling. (Dave Borowitz) + + FEATURES + + * Add include-tag capability to server. (Dave Borowitz) + + * New dulwich.fastexport module that can generate fastexport + streams. (Jelmer Vernooij) + + * Implemented BaseRepo.__contains__. (Jelmer Vernooij) + + * Add __setitem__ to DictRefsContainer. (Dave Borowitz) + + * Overall improvements checking Git objects. (Dave Borowitz) + + * Packs are now verified while they are received. (Dave Borowitz) + + TESTS + + * Add framework for testing compatibility with C Git. (Dave Borowitz) + + * Add various tests for the use of non-bare repositories. (Dave Borowitz) + + * Cope with diffstat not being available on all platforms. + (Tay Ray Chuan, Jelmer Vernooij) + + * Add make_object and make_commit convenience functions to test utils. + (Dave Borowitz) + + API BREAKAGES + + * The 'committer' and 'message' arguments to Repo.do_commit() have + been swapped. 'committer' is now optional. (Jelmer Vernooij) + + * Repo.get_blob, Repo.commit, Repo.tag and Repo.tree are now deprecated. + (Jelmer Vernooij) + + * RefsContainer.set_ref() was renamed to RefsContainer.set_symbolic_ref(), + for clarity. (Jelmer Vernooij) + + API CHANGES + + * The primary serialization APIs in dulwich.objects now work + with chunks of strings rather than with full-text strings. + (Jelmer Vernooij) + +0.5.02010-03-03 + + BUG FIXES + + * Support custom fields in commits (readonly). (Jelmer Vernooij) + + * Improved ref handling. (Dave Borowitz) + + * Rework server protocol to be smarter and interoperate with cgit client. + (Dave Borowitz) + + * Add a GitFile class that uses the same locking protocol for writes as + cgit. (Dave Borowitz) + + * Cope with forward slashes correctly in the index on Windows. + (Jelmer Vernooij, #526793) + + FEATURES + + * --pure option to setup.py to allow building/installing without the C + extensions. (Hal Wine, Anatoly Techtonik, Jelmer Vernooij, #434326) + + * Implement Repo.get_config(). (Jelmer Vernooij, Augie Fackler) + + * HTTP dumb and smart server. (Dave Borowitz) + + * Add abstract baseclass for Repo that does not require file system + operations. (Dave Borowitz) + +0.4.1 2010-01-03 + + FEATURES + + * Add ObjectStore.iter_tree_contents(). (Jelmer Vernooij) + + * Add Index.changes_from_tree(). (Jelmer Vernooij) + + * Add ObjectStore.tree_changes(). (Jelmer Vernooij) + + * Add functionality for writing patches in dulwich.patch. + (Jelmer Vernooij) + +0.4.0 2009-10-07 + + DOCUMENTATION + + * Added tutorial. + + API CHANGES + + * dulwich.object_store.tree_lookup_path will now return the mode and + sha of the object found rather than the object itself. + + BUG FIXES + + * Use binascii.hexlify / binascii.unhexlify for better performance. + + * Cope with extra unknown data in index files by ignoring it (for now). + + * Add proper error message when server unexpectedly hangs up. (#415843) + + * Correctly write opcode for equal in create_delta. + +0.3.3 2009-07-23 + + FEATURES + + * Implement ShaFile.__hash__(). + + * Implement Tree.__len__() + + BUG FIXES + + * Check for 'objects' and 'refs' directories + when looking for a Git repository. (#380818) + +0.3.2 2009-05-20 + + BUG FIXES + + * Support the encoding field in Commits. + + * Some Windows compatibility fixes. + + * Fixed several issues in commit support. + + FEATURES + + * Basic support for handling submodules. + +0.3.1 2009-05-13 + + FEATURES + + * Implemented Repo.__getitem__, Repo.__setitem__ and Repo.__delitem__ to + access content. + + API CHANGES + + * Removed Repo.set_ref, Repo.remove_ref, Repo.tags, Repo.get_refs and + Repo.heads in favor of Repo.refs, a dictionary-like object for accessing + refs. + + BUG FIXES + + * Removed import of 'sha' module in objects.py, which was causing + deprecation warnings on Python 2.6. + +0.3.0 2009-05-10 + + FEATURES + + * A new function 'commit_tree' has been added that can commit a tree + based on an index. + + BUG FIXES + + * The memory usage when generating indexes has been significantly reduced. + + * A memory leak in the C implementation of parse_tree has been fixed. + + * The send-pack smart server command now works. (Thanks Scott Chacon) + + * The handling of short timestamps (less than 10 digits) has been fixed. + + * The handling of timezones has been fixed. + +0.2.1 2009-04-30 + + BUG FIXES + + * Fix compatibility with Python2.4. + +0.2.0 2009-04-30 + + FEATURES + + * Support for activity reporting in smart protocol client. + + * Optional C extensions for better performance in a couple of + places that are performance-critical. + +0.1.1 2009-03-13 + + BUG FIXES + + * Fixed regression in Repo.find_missing_objects() + + * Don't fetch ^{} objects from remote hosts, as requesting them + causes a hangup. + + * Always write pack to disk completely before calculating checksum. + + FEATURES + + * Allow disabling thin packs when talking to remote hosts. + +0.1.0 2009-01-24 + + * Initial release. diff --git a/lib/dulwich-0.9.1/PKG-INFO b/lib/dulwich-0.9.1/PKG-INFO new file mode 100644 --- /dev/null +++ b/lib/dulwich-0.9.1/PKG-INFO @@ -0,0 +1,18 @@ +Metadata-Version: 1.0 +Name: dulwich +Version: 0.9.1 +Summary: Python Git Library +Home-page: http://samba.org/~jelmer/dulwich +Author: Jelmer Vernooij +Author-email: jelmer at samba.org +License: GPLv2 or later +Description: + Simple Python implementation of the Git file formats and + protocols. Dulwich is the place where Mr. and Mrs. Git live + in one of the Monty Python sketches. + + All functionality is available in pure Python, but (optional) + C extensions are also available for better performance. + +Keywords: git +Platform: UNKNOWN diff --git a/lib/dulwich-0.9.1/README b/lib/dulwich-0.9.1/README new file mode 100644 --- /dev/null +++ b/lib/dulwich-0.9.1/README @@ -0,0 +1,21 @@ +This is the Dulwich project. + +It aims to give an interface to git repos (both local and remote) that doesn't +call out to git directly but instead uses pure Python. + +The project is named after the part of London that Mr. and Mrs. Git live in +in the particular Monty Python sketch. It is based on the Python-Git module +that James Westby released in 2007 and now +maintained by Jelmer Vernooij et al. + +Please file bugs in the Dulwich project on Launchpad: + +https://bugs.launchpad.net/dulwich/+filebug + +The dulwich documentation can be found in doc/ and on the web: + +http://www.samba.org/~jelmer/dulwich/docs/ + +The API reference can be generated using pydoctor, by running "make pydoctor", or on the web: + +http://www.samba.org/~jelmer/dulwich/apidocs diff --git a/lib/dulwich-0.9.1/bin/dul-daemon b/lib/dulwich-0.9.1/bin/dul-daemon new file mode 100755 --- /dev/null +++ b/lib/dulwich-0.9.1/bin/dul-daemon @@ -0,0 +1,23 @@ +#!/usr/bin/python +# dul-daemon - Simple git-daemon-like server +# Copyright (C) 2008 John Carr +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; version 2 +# or (at your option) a later version of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, +# MA 02110-1301, USA. + +from dulwich.server import main + +if __name__ == '__main__': + main() diff --git a/lib/dulwich-0.9.1/bin/dul-receive-pack b/lib/dulwich-0.9.1/bin/dul-receive-pack new file mode 100755 --- /dev/null +++ b/lib/dulwich-0.9.1/bin/dul-receive-pack @@ -0,0 +1,28 @@ +#!/usr/bin/python +# dul-receive-pack - git-receive-pack in python +# Copyright (C) 2008 John Carr +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; version 2 +# or (at your option) a later version of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, +# MA 02110-1301, USA. + +from dulwich.server import serve_command, ReceivePackHandler +import os +import sys + +if len(sys.argv) < 2: + print >>sys.stderr, "usage: %s " % os.path.basename(sys.argv[0]) + sys.exit(1) + +sys.exit(serve_command(ReceivePackHandler)) diff --git a/lib/dulwich-0.9.1/bin/dul-upload-pack b/lib/dulwich-0.9.1/bin/dul-upload-pack new file mode 100755 --- /dev/null +++ b/lib/dulwich-0.9.1/bin/dul-upload-pack @@ -0,0 +1,28 @@ +#!/usr/bin/python +# dul-upload-pack - git-upload-pack in python +# Copyright (C) 2008 John Carr +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; version 2 +# or (at your option) a later version of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, +# MA 02110-1301, USA. + +from dulwich.server import serve_command, UploadPackHandler +import os +import sys + +if len(sys.argv) < 2: + print >>sys.stderr, "usage: %s " % os.path.basename(sys.argv[0]) + sys.exit(1) + +sys.exit(serve_command(UploadPackHandler)) diff --git a/lib/dulwich-0.9.1/bin/dul-web b/lib/dulwich-0.9.1/bin/dul-web new file mode 100644 --- /dev/null +++ b/lib/dulwich-0.9.1/bin/dul-web @@ -0,0 +1,23 @@ +#!/usr/bin/python +# dul-web - HTTP-based git server +# Copyright (C) 2010 Google, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; version 2 +# or (at your option) a later version of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, +# MA 02110-1301, USA. + +from dulwich.web import main + +if __name__ == '__main__': + main() diff --git a/lib/dulwich-0.9.1/bin/dulwich b/lib/dulwich-0.9.1/bin/dulwich new file mode 100755 --- /dev/null +++ b/lib/dulwich-0.9.1/bin/dulwich @@ -0,0 +1,232 @@ +#!/usr/bin/python -u +# +# dulwich - Simple command-line interface to Dulwich +# Copyright (C) 2008-2011 Jelmer Vernooij +# vim: expandtab +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; version 2 +# or (at your option) a later version of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, +# MA 02110-1301, USA. + +"""Simple command-line interface to Dulwich> + +This is a very simple command-line wrapper for Dulwich. It is by +no means intended to be a full-blown Git command-line interface but just +a way to test Dulwich. +""" + +import os +import sys +from getopt import getopt + +from dulwich.client import get_transport_and_path +from dulwich.errors import ApplyDeltaError +from dulwich.index import Index +from dulwich.pack import Pack, sha_to_hex +from dulwich.patch import write_tree_diff +from dulwich.repo import Repo +from dulwich.server import update_server_info + + +def cmd_archive(args): + opts, args = getopt(args, "", []) + client, path = get_transport_and_path(args.pop(0)) + committish = args.pop(0) + client.archive(path, committish, sys.stdout.write, sys.stderr.write) + + +def cmd_fetch_pack(args): + opts, args = getopt(args, "", ["all"]) + opts = dict(opts) + client, path = get_transport_and_path(args.pop(0)) + r = Repo(".") + if "--all" in opts: + determine_wants = r.object_store.determine_wants_all + else: + determine_wants = lambda x: [y for y in args if not y in r.object_store] + client.fetch(path, r, determine_wants) + + +def cmd_fetch(args): + opts, args = getopt(args, "", []) + opts = dict(opts) + client, path = get_transport_and_path(args.pop(0)) + r = Repo(".") + if "--all" in opts: + determine_wants = r.object_store.determine_wants_all + refs = client.fetch(path, r, progress=sys.stdout.write) + print "Remote refs:" + for item in refs.iteritems(): + print "%s -> %s" % item + + +def cmd_log(args): + opts, args = getopt(args, "", []) + if len(args) > 0: + path = args.pop(0) + else: + path = "." + r = Repo(path) + todo = [r.head()] + done = set() + while todo: + sha = todo.pop() + assert isinstance(sha, str) + if sha in done: + continue + done.add(sha) + commit = r[sha] + print "-" * 50 + print "commit: %s" % sha + if len(commit.parents) > 1: + print "merge: %s" % "...".join(commit.parents[1:]) + print "author: %s" % commit.author + print "committer: %s" % commit.committer + print "" + print commit.message + print "" + todo.extend([p for p in commit.parents if p not in done]) + + +def cmd_diff(args): + opts, args = getopt(args, "", []) + + if args == []: + print "Usage: dulwich diff COMMITID" + sys.exit(1) + + r = Repo(".") + commit_id = args[0] + commit = r[commit_id] + parent_commit = r[commit.parents[0]] + write_tree_diff(sys.stdout, r.object_store, parent_commit.tree, commit.tree) + + +def cmd_dump_pack(args): + opts, args = getopt(args, "", []) + + if args == []: + print "Usage: dulwich dump-pack FILENAME" + sys.exit(1) + + basename, _ = os.path.splitext(args[0]) + x = Pack(basename) + print "Object names checksum: %s" % x.name() + print "Checksum: %s" % sha_to_hex(x.get_stored_checksum()) + if not x.check(): + print "CHECKSUM DOES NOT MATCH" + print "Length: %d" % len(x) + for name in x: + try: + print "\t%s" % x[name] + except KeyError, k: + print "\t%s: Unable to resolve base %s" % (name, k) + except ApplyDeltaError, e: + print "\t%s: Unable to apply delta: %r" % (name, e) + + +def cmd_dump_index(args): + opts, args = getopt(args, "", []) + + if args == []: + print "Usage: dulwich dump-index FILENAME" + sys.exit(1) + + filename = args[0] + idx = Index(filename) + + for o in idx: + print o, idx[o] + + +def cmd_init(args): + opts, args = getopt(args, "", ["bare"]) + opts = dict(opts) + + if args == []: + path = os.getcwd() + else: + path = args[0] + + if not os.path.exists(path): + os.mkdir(path) + + if "--bare" in opts: + Repo.init_bare(path) + else: + Repo.init(path) + + +def cmd_clone(args): + opts, args = getopt(args, "", []) + opts = dict(opts) + + if args == []: + print "usage: dulwich clone host:path [PATH]" + sys.exit(1) + client, host_path = get_transport_and_path(args.pop(0)) + + if len(args) > 0: + path = args.pop(0) + else: + path = host_path.split("/")[-1] + + if not os.path.exists(path): + os.mkdir(path) + r = Repo.init(path) + remote_refs = client.fetch(host_path, r, + determine_wants=r.object_store.determine_wants_all, + progress=sys.stdout.write) + r["HEAD"] = remote_refs["HEAD"] + + +def cmd_commit(args): + opts, args = getopt(args, "", ["message"]) + opts = dict(opts) + r = Repo(".") + committer = "%s <%s>" % (os.getenv("GIT_COMMITTER_NAME"), + os.getenv("GIT_COMMITTER_EMAIL")) + author = "%s <%s>" % (os.getenv("GIT_AUTHOR_NAME"), + os.getenv("GIT_AUTHOR_EMAIL")) + r.do_commit(committer=committer, author=author, message=opts["--message"]) + + +def cmd_update_server_info(args): + r = Repo(".") + update_server_info(r) + + +commands = { + "commit": cmd_commit, + "fetch-pack": cmd_fetch_pack, + "fetch": cmd_fetch, + "dump-pack": cmd_dump_pack, + "dump-index": cmd_dump_index, + "init": cmd_init, + "log": cmd_log, + "clone": cmd_clone, + "archive": cmd_archive, + "update-server-info": cmd_update_server_info, + "diff": cmd_diff, + } + +if len(sys.argv) < 2: + print "Usage: %s <%s> [OPTIONS...]" % (sys.argv[0], "|".join(commands.keys())) + sys.exit(1) + +cmd = sys.argv[1] +if not cmd in commands: + print "No such subcommand: %s" % cmd + sys.exit(1) +commands[cmd](sys.argv[2:]) diff --git a/lib/dulwich-0.9.1/dist/dulwich-0.9.1-py2.7-linux-x86_64.egg b/lib/dulwich-0.9.1/dist/dulwich-0.9.1-py2.7-linux-x86_64.egg new file mode 100644 index 0000000000000000000000000000000000000000..7768835f0a496224f916bbd51a62fda4b32905e6 GIT binary patch [cut] diff --git a/lib/dulwich-0.9.1/dist/dulwich-0.9.1-py2.7.egg b/lib/dulwich-0.9.1/dist/dulwich-0.9.1-py2.7.egg new file mode 100644 index 0000000000000000000000000000000000000000..3ce6ca8f51814c27ca6a42c8273298ac70a44348 GIT binary patch [cut] diff --git a/lib/dulwich-0.9.1/docs/Makefile b/lib/dulwich-0.9.1/docs/Makefile new file mode 100644 --- /dev/null +++ b/lib/dulwich-0.9.1/docs/Makefile @@ -0,0 +1,96 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " pdf to make PDF document" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/dulwich.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/dulwich.qhc" + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ + "run these through (pdf)latex." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +pdf: + $(SPHINXBUILD) -b pdf $(ALLSPHINXOPTS) $(BUILDDIR)/pdf + @echo + @echo "Build finished. The PDF files are in $(BUILDDIR)/pdf." + diff --git a/lib/dulwich-0.9.1/docs/conf.py b/lib/dulwich-0.9.1/docs/conf.py new file mode 100644 --- /dev/null +++ b/lib/dulwich-0.9.1/docs/conf.py @@ -0,0 +1,215 @@ +# -*- coding: utf-8 -*- +# +# dulwich documentation build configuration file, created by +# sphinx-quickstart on Thu Feb 18 23:18:28 2010. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys, os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath('..')) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__)))) +dulwich = __import__('dulwich') + +# -- General configuration ----------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc'] +try: + import rst2pdf + if rst2pdf.version >= '0.16': + extensions.append('rst2pdf.pdfbuilder') +except ImportError: + print "[NOTE] In order to build PDF you need rst2pdf with version >=0.16" + + +autoclass_content = "both" + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['templates'] + +# The suffix of source filenames. +source_suffix = '.txt' + +# The encoding of source files. +#source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'dulwich' +copyright = u'2011, Jelmer Vernooij' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '.'.join(map(str, dulwich.__version__[:2])) +# The full version, including alpha/beta/rc tags. +release = '.'.join(map(str, dulwich.__version__)) + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of documents that shouldn't be included in the build. +#unused_docs = [] + +# List of directories, relative to source directory, that shouldn't be searched +# for source files. +exclude_trees = ['build'] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +#html_theme = 'default' +html_theme = 'nature' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the From noreply at buildbot.pypy.org Wed Apr 8 11:49:43 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 8 Apr 2015 11:49:43 +0200 (CEST) Subject: [pypy-commit] benchmarks single-run: improve the json format Message-ID: <20150408094943.BC9831C0EEE@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: single-run Changeset: r322:fc84a3202e27 Date: 2015-04-08 11:49 +0200 http://bitbucket.org/pypy/benchmarks/changeset/fc84a3202e27/ Log: improve the json format diff --git a/bench-data.json b/bench-data.json --- a/bench-data.json +++ b/bench-data.json @@ -9,6 +9,19 @@ }, "bm_mako": { }, + "bm_dulwich_log": { + }, + "bm_icbd": { + }, + "bm_krakatau": { + }, + "bm_mdp": { + }, + "deltablue": { + }, + "pypy_interp": { + "description": "interpreting py.py" + }, "chaos": { "description": "Creates chaosgame-like fractals" }, diff --git a/runner.py b/runner.py --- a/runner.py +++ b/runner.py @@ -6,6 +6,7 @@ import socket import sys import os +import time import benchmarks from saveresults import save @@ -18,10 +19,20 @@ class WrongBenchmark(Exception): pass +def convert_results(result_list): + r = [] + for bench, cls, dict, t0 in result_list: + runs = [] + cur_time = t0 + for t in dict['times']: + runs.append({"start_timestamp": cur_time, "duration": t}) + cur_time += t + r.append({"name": bench, "runs": runs, "events": {}}) + return r def run_and_store(benchmark_set, result_filename, path, revision=0, options='', branch='default', args='', upload=False, - fast=False, full_store=False): + fast=False, full_store=False, parser_options=None): _funcs = perf.BENCH_FUNCS.copy() _funcs.update(perf._FindAllBenchmarks(benchmarks.__dict__)) bench_data = json.load(open('bench-data.json')) @@ -38,16 +49,24 @@ if full_store: opts += ['--no_statistics'] opts += [path] + start_time = time.time() results = perf.main(opts, funcs) + end_time = time.time() f = open(str(result_filename), "w") - results = [(name, result.__class__.__name__, result.__dict__) - for name, result in results] + results = [(name, result.__class__.__name__, result.__dict__, t0) + for name, result, t0 in results] + force_host = parser_options.force_host f.write(json.dumps({ 'revision': revision, - 'results': results, + 'results': convert_results(results), + "interpreter": parser_options.python, + "machine": force_host if force_host else socket.gethostname(), + "protocol_version_no": "1", + "start_timestamp": start_time, + "end_timestamp": end_time, 'options': options, 'branch': branch, - })) + }, indent=4)) f.close() return results @@ -168,7 +187,8 @@ results = run_and_store(benchmarks, output_filename, path, revision, args=args, fast=fast, - full_store=full_store, branch=branch) + full_store=full_store, branch=branch, + parser_options=options) if options.upload_url: branch = options.upload_branch or 'default' diff --git a/unladen_swallow/perf.py b/unladen_swallow/perf.py --- a/unladen_swallow/perf.py +++ b/unladen_swallow/perf.py @@ -779,13 +779,8 @@ return SimpleBenchmark(MeasureDjango, *args, **kwargs) -<<<<<<< local def MeasureRietveld(python, options, bench_data): - PYTHONPATH = ":".join([DJANGO_DIR, -======= -def MeasureRietveld(python, options): PYTHONPATH = os.pathsep.join([DJANGO_DIR, ->>>>>>> other # These paths are lifted from # lib/google_appengine.appcfg.py. Note that we use # our own version of Django instead of Appengine's. @@ -1133,6 +1128,7 @@ for name in sorted(should_run): func, bench_data = bench_funcs[name] print "Running %s..." % name + t0 = time.time() # PyPy specific modification: let the func to return a list of results # for sub-benchmarks bench_result = func(base_cmd_prefix, options, bench_data) @@ -1140,15 +1136,15 @@ if isinstance(bench_result, list): for subname, subresult in bench_result: fullname = '%s_%s' % (name, subname) - results.append((fullname, subresult)) + results.append((fullname, subresult, t0)) else: - results.append((name, bench_result)) + results.append((name, bench_result, t0)) print print "Report on %s" % " ".join(platform.uname()) if multiprocessing: print "Total CPU cores:", multiprocessing.cpu_count() - for name, result in results: + for name, result, start_time in results: print print "###", name, "###" print result.string_representation() From noreply at buildbot.pypy.org Wed Apr 8 11:53:12 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 8 Apr 2015 11:53:12 +0200 (CEST) Subject: [pypy-commit] benchmarks single-run: events is a list not a dict Message-ID: <20150408095312.0FA181C0173@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: single-run Changeset: r323:cd369a185e0a Date: 2015-04-08 11:53 +0200 http://bitbucket.org/pypy/benchmarks/changeset/cd369a185e0a/ Log: events is a list not a dict diff --git a/runner.py b/runner.py --- a/runner.py +++ b/runner.py @@ -27,7 +27,7 @@ for t in dict['times']: runs.append({"start_timestamp": cur_time, "duration": t}) cur_time += t - r.append({"name": bench, "runs": runs, "events": {}}) + r.append({"name": bench, "runs": runs, "events": []}) return r def run_and_store(benchmark_set, result_filename, path, revision=0, From noreply at buildbot.pypy.org Wed Apr 8 12:17:04 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 8 Apr 2015 12:17:04 +0200 (CEST) Subject: [pypy-commit] benchmarks single-run: tweak the json Message-ID: <20150408101704.926721C1151@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: single-run Changeset: r324:aac968027850 Date: 2015-04-08 12:14 +0200 http://bitbucket.org/pypy/benchmarks/changeset/aac968027850/ Log: tweak the json diff --git a/runner.py b/runner.py --- a/runner.py +++ b/runner.py @@ -58,8 +58,8 @@ force_host = parser_options.force_host f.write(json.dumps({ 'revision': revision, - 'results': convert_results(results), - "interpreter": parser_options.python, + 'benchmarks': convert_results(results), + "interpreter": parser_options.interpreter_name or parser_options.python, "machine": force_host if force_host else socket.gethostname(), "protocol_version_no": "1", "start_timestamp": start_time, @@ -110,6 +110,11 @@ 'will be store in the result json and used for the upload. ' "(default: 'default')")) benchmark_group.add_option( + '--force-interpreter-name', default=None, action='store', + dest='interpreter_name', + help=("Force the interpreter name present",) + ) + benchmark_group.add_option( '-r', '--revision', action="store", dest='upload_revision', help=("Specify the revision of the 'changed' interpreter. " From noreply at buildbot.pypy.org Wed Apr 8 12:17:16 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 8 Apr 2015 12:17:16 +0200 (CEST) Subject: [pypy-commit] buildbot default: force interpreter name Message-ID: <20150408101716.2C9A61C1151@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r947:2c1f75239406 Date: 2015-04-08 12:17 +0200 http://bitbucket.org/pypy/buildbot/changeset/2c1f75239406/ Log: force interpreter name diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -737,9 +737,9 @@ description="run benchmarks on top of pypy-c", command=["python", "runner.py", '--output-filename', 'result.json', '--python', pypy_c_rel, - '--full-store', '--revision', WithProperties('%(got_revision)s'), '--branch', WithProperties('%(branch)s'), + '--force-interpreter-name', 'pypy-c-jit', ], workdir='./benchmarks', timeout=3600)) From noreply at buildbot.pypy.org Wed Apr 8 12:24:06 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 8 Apr 2015 12:24:06 +0200 (CEST) Subject: [pypy-commit] buildbot default: enable the speed slave Message-ID: <20150408102406.455D51C116D@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r948:d1fa1ac08e0f Date: 2015-04-08 12:24 +0200 http://bitbucket.org/pypy/buildbot/changeset/d1fa1ac08e0f/ Log: enable the speed slave diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -226,12 +226,6 @@ 'factory' : pypyJITTranslatedTestFactoryFreeBSD, "category": 'freebsd64' }, - {"name": JITBENCH64_NEW, - "slavenames": [], - "builddir": JITBENCH64_NEW, - "factory": pypyJITBenchmarkFactory64_speed, - "category": "benchmark-run", - }, # openindiana {'name': JITINDIANA32, 'slavenames': [], @@ -280,7 +274,7 @@ Nightly("nightly-1-00", [ JITBENCH, # on tannit32, uses 1 core (in part exclusively) JITBENCH64, # on tannit64, uses 1 core (in part exclusively) - #JITBENCH64_NEW, # on speed64, uses 1 core (in part exclusively) + JITBENCH64_NEW, # on speed64, uses 1 core (in part exclusively) ], branch=None, hour=1, minute=0), @@ -327,7 +321,7 @@ JITONLYLINUXPPC64, JITBENCH, JITBENCH64, - #JITBENCH64_NEW, + JITBENCH64_NEW, NUMPY_64, NUMPY_WIN, #INDIANA32, @@ -420,6 +414,12 @@ "category": "benchmark-run", # the locks are acquired with fine grain inside the build }, + {"name": JITBENCH64_NEW, + "slavenames": ['speed-old'], + "builddir": JITBENCH64_NEW, + "factory": pypyJITBenchmarkFactory64_speed, + "category": "benchmark-run", + }, {"name": MACOSX32, "slavenames": ["minime"], "builddir": MACOSX32, From noreply at buildbot.pypy.org Wed Apr 8 16:53:48 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 8 Apr 2015 16:53:48 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Starting to reuse some code from creflect for parsing C types Message-ID: <20150408145348.B41CD1C14BF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1682:bd1b2331850e Date: 2015-04-08 16:54 +0200 http://bitbucket.org/cffi/cffi/changeset/bd1b2331850e/ Log: Starting to reuse some code from creflect for parsing C types diff --git a/new/parse_c_type.c b/new/parse_c_type.c new file mode 100644 --- /dev/null +++ b/new/parse_c_type.c @@ -0,0 +1,590 @@ +#include +#include +#include "parse_c_type.h" + + +enum token_e { + TOK_STAR='*', + TOK_OPEN_PAREN='(', + TOK_CLOSE_PAREN=')', + TOK_OPEN_BRACKET='[', + TOK_CLOSE_BRACKET=']', + TOK_COMMA=',', + + TOK_START=256, + TOK_END, + TOK_ERROR, + TOK_IDENTIFIER, + TOK_INTEGER, + TOK_DOTDOTDOT, + + /* keywords */ + TOK__BOOL, + TOK_CHAR, + //TOK__COMPLEX, + TOK_CONST, + TOK_DOUBLE, + TOK_FLOAT, + //TOK__IMAGINARY, + TOK_INT, + TOK_LONG, + TOK_SHORT, + TOK_SIGNED, + TOK_STRUCT, + TOK_UNION, + TOK_UNSIGNED, + TOK_VOID, + TOK_VOLATILE, +}; + +typedef struct { + enum token_e kind; + const char *p, **error_location, **error_message; + size_t size; + ctype_opcode_t *opcodes, *opcodes_end; +} token_t; + +static int is_space(char x) +{ + return (x == ' ' || x == '\f' || x == '\n' || x == '\r' || + x == '\t' || x == '\v'); +} + +static int is_ident_first(char x) +{ + return (('A' <= x && x <= 'Z') || ('a' <= x && x <= 'z') || x == '_'); +} + +static int is_digit(char x) +{ + return ('0' <= x && x <= '9'); +} + +static int is_ident_next(char x) +{ + return (is_ident_first(x) || is_digit(x)); +} + +static char get_following_char(token_t *tok) +{ + const char *p = tok->p + tok->size; + if (tok->kind == TOK_ERROR) + return 0; + while (is_space(*p)) + p++; + return *p; +} + +static void next_token(token_t *tok) +{ + const char *p = tok->p + tok->size; + if (tok->kind == TOK_ERROR) + return; + while (!is_ident_first(*p)) { + if (is_space(*p)) { + p++; + } + else if (is_digit(*p)) { + tok->kind = TOK_INTEGER; + tok->p = p; + tok->size = 1; + while (is_digit(p[tok->size])) + tok->size++; + return; + } + else if (p[0] == '.' && p[1] == '.' && p[2] == '.') { + tok->kind = TOK_DOTDOTDOT; + tok->p = p; + tok->size = 3; + return; + } + else if (*p) { + tok->kind = *p; + tok->p = p; + tok->size = 1; + return; + } + else { + tok->kind = TOK_END; + tok->p = p; + tok->size = 0; + return; + } + } + tok->kind = TOK_IDENTIFIER; + tok->p = p; + tok->size = 1; + while (is_ident_next(p[tok->size])) + tok->size++; + + switch (*p) { + case '_': + if (tok->size == 5 && !memcmp(p, "_Bool", 5)) tok->kind = TOK__BOOL; + break; + case 'c': + if (tok->size == 4 && !memcmp(p, "char", 4)) tok->kind = TOK_CHAR; + if (tok->size == 5 && !memcmp(p, "const", 5)) tok->kind = TOK_CONST; + break; + case 'd': + if (tok->size == 6 && !memcmp(p, "double", 6)) tok->kind = TOK_DOUBLE; + break; + case 'f': + if (tok->size == 5 && !memcmp(p, "float", 5)) tok->kind = TOK_FLOAT; + break; + case 'i': + if (tok->size == 3 && !memcmp(p, "int", 3)) tok->kind = TOK_INT; + break; + case 'l': + if (tok->size == 4 && !memcmp(p, "long", 4)) tok->kind = TOK_LONG; + break; + case 's': + if (tok->size == 5 && !memcmp(p, "short", 5)) tok->kind = TOK_SHORT; + if (tok->size == 6 && !memcmp(p, "signed", 6)) tok->kind = TOK_SIGNED; + if (tok->size == 6 && !memcmp(p, "struct", 6)) tok->kind = TOK_STRUCT; + break; + case 'u': + if (tok->size == 5 && !memcmp(p, "union", 5)) tok->kind = TOK_UNION; + if (tok->size == 8 && !memcmp(p,"unsigned",8)) tok->kind = TOK_UNSIGNED; + break; + case 'v': + if (tok->size == 4 && !memcmp(p, "void", 4)) tok->kind = TOK_VOID; + if (tok->size == 8 && !memcmp(p,"volatile",8)) tok->kind = TOK_VOLATILE; + break; + } +} + +static void parse_error(token_t *tok, const char *msg) +{ + if (tok->kind != TOK_ERROR) { + tok->kind = TOK_ERROR; + if (tok->error_location) + *tok->error_location = tok->p; + if (tok->error_message) + *tok->error_message = msg; + } +} + +static ctype_opcode_t *alloc_ds(token_t *tok, size_t num) +{ + ctype_opcode_t *result = tok->opcodes; + if (num > tok->opcodes_end - result) { + parse_error(tok, "type too lengthy"); + return NULL; + } + tok->opcodes += num; + return result; +} + +#if 0 +static void parse_complete(token_t *tok, _crx_qual_type *result); + +static void parse_sequel(token_t *tok, intptr_t ds_end) +{ + intptr_t *ds; + while (tok->kind == TOK_STAR || tok->kind == TOK_CONST || + tok->kind == TOK_VOLATILE) { + ds = alloc_ds(tok, 1); + if (ds == NULL) + return; + ds[0] = tok->kind; + next_token(tok); + } + + int check_for_grouping = -1; + if (tok->kind == TOK_IDENTIFIER) { + next_token(tok); /* skip a potential variable name */ + check_for_grouping = 1; + } + + intptr_t *jump_slot = alloc_ds(tok, 1); + if (jump_slot == NULL) + return; + *jump_slot = ds_end; + + next_right_part: + check_for_grouping++; + + switch (tok->kind) { + + case TOK_OPEN_PAREN: + next_token(tok); + + if (check_for_grouping == 0 && (tok->kind == TOK_STAR || + tok->kind == TOK_CONST || + tok->kind == TOK_VOLATILE || + tok->kind == TOK_OPEN_BRACKET)) { + /* just parentheses for grouping */ + ds = tok->delay_slots; + parse_sequel(tok, *jump_slot); + *jump_slot = -(ds - tok->all_delay_slots); + } + else { + /* function type */ + ds = alloc_ds(tok, 2); + if (ds == NULL) + return; + ds[0] = TOK_OPEN_PAREN; + ds[1] = 0; + if (tok->kind == TOK_VOID && get_following_char(tok) == ')') { + next_token(tok); + } + if (tok->kind != TOK_CLOSE_PAREN) { + while (1) { + if (tok->kind == TOK_DOTDOTDOT) { + ds[0] = TOK_DOTDOTDOT; + next_token(tok); + break; + } + intptr_t *ds_type = alloc_ds(tok, 2); + if (ds_type == NULL) + return; + assert(ds_type == ds + 2 + 2 * ds[1]); + assert(2 * sizeof(intptr_t) >= sizeof(_crx_qual_type)); + parse_complete(tok, (_crx_qual_type *)ds_type); + ds[1]++; + if (tok->kind != TOK_COMMA) + break; + next_token(tok); + } + } + intptr_t *ds_next = alloc_ds(tok, 1); + if (ds_next == NULL) + return; + assert(ds_next == ds + 2 + 2 * ds[1]); + *ds_next = *jump_slot; + *jump_slot = -(ds - tok->all_delay_slots); + } + + if (tok->kind != TOK_CLOSE_PAREN) { + parse_error(tok, "expected ')'"); + return; + } + next_token(tok); + goto next_right_part; + + case TOK_OPEN_BRACKET: + { + uintptr_t length = (uintptr_t)-1; + next_token(tok); + if (tok->kind != TOK_CLOSE_BRACKET) { + if (tok->kind != TOK_INTEGER) { + parse_error(tok, "expected a positive integer constant"); + return; + } + + if (sizeof(uintptr_t) > sizeof(unsigned long)) + length = strtoull(tok->p, NULL, 10); + else + length = strtoul(tok->p, NULL, 10); + if (length == (uintptr_t)-1) { + parse_error(tok, "number too large"); + return; + } + next_token(tok); + } + + if (tok->kind != TOK_CLOSE_BRACKET) { + parse_error(tok, "expected ']'"); + return; + } + next_token(tok); + + ds = alloc_ds(tok, 3); + if (ds == NULL) + return; + ds[0] = TOK_OPEN_BRACKET; + ds[1] = (intptr_t)length; + ds[2] = *jump_slot; + *jump_slot = -(ds - tok->all_delay_slots); + goto next_right_part; + } + default: + break; + } +} +#endif + +#if 0 +static void fetch_delay_slots(token_t *tok, _crx_qual_type *result, + intptr_t *delay_slot) +{ + if (tok->kind == TOK_ERROR) + return; + tok->delay_slots = delay_slot; + while (1) { + intptr_t tok_kind = *delay_slot++; + if (tok_kind <= 0) { + delay_slot = tok->all_delay_slots + (-tok_kind); + continue; + } + switch (tok_kind) { + case TOK_END: + return; /* done */ + case TOK_STAR: + result->type = tok->cb->get_pointer_type(tok->cb, result->type, + result->qualifiers); + result->qualifiers = 0; + break; + case TOK_CONST: + result->qualifiers |= _CRX_CONST; + break; + case TOK_VOLATILE: + result->qualifiers |= _CRX_VOLATILE; + break; + case TOK_OPEN_BRACKET: /* array */ + { + uintptr_t length = (uintptr_t)*delay_slot++; + if (length != (uintptr_t)-1) + result->type = tok->cb->get_array_type( + tok->cb, result->type, length); + else + result->type = tok->cb->get_incomplete_array_type( + tok->cb, result->type); + /* result->qualifiers remains unmodified */ + break; + } + case TOK_OPEN_PAREN: /* function */ + case TOK_DOTDOTDOT: /* function ending with a '...' */ + { + intptr_t nbargs = *delay_slot++; + _crx_type_t *t1; + _crx_qual_type *argtypes = (_crx_qual_type *)delay_slot; + delay_slot += 2 * nbargs; + if (tok_kind == TOK_DOTDOTDOT) + t1 = tok->cb->get_ellipsis_function_type(tok->cb, + result->type, + argtypes, nbargs); + else + t1 = tok->cb->get_function_type(tok->cb, result->type, + argtypes, nbargs, NULL); + result->type = t1; + result->qualifiers = 0; /* drop qualifiers on the return type */ + break; + } + default: + assert(!"missing delay slot case"); + } + } +} +#endif + +static void parse_complete(token_t *tok) +{ + int const_qualifier = 0, volatile_qualifier = 0; + + qualifiers: + switch (tok->kind) { + case TOK_CONST: + const_qualifier = 1; + next_token(tok); + goto qualifiers; + case TOK_VOLATILE: + volatile_qualifier = 1; + next_token(tok); + goto qualifiers; + default: + ; + } + + int t1; + int modifiers_length = 0; + int modifiers_sign = 0; + modifiers: + switch (tok->kind) { + + case TOK_SHORT: + if (modifiers_length != 0) { + parse_error(tok, "'short' after another 'short' or 'long'"); + return; + } + modifiers_length--; + next_token(tok); + goto modifiers; + + case TOK_LONG: + if (modifiers_length < 0) { + parse_error(tok, "'long' after 'short'"); + return; + } + if (modifiers_length >= 2) { + parse_error(tok, "'long long long' is too long"); + return; + } + modifiers_length++; + next_token(tok); + goto modifiers; + + case TOK_SIGNED: + if (modifiers_sign) { + parse_error(tok, "multiple 'signed' or 'unsigned'"); + return; + } + modifiers_sign++; + next_token(tok); + goto modifiers; + + case TOK_UNSIGNED: + if (modifiers_sign) { + parse_error(tok, "multiple 'signed' or 'unsigned'"); + return; + } + modifiers_sign--; + next_token(tok); + goto modifiers; + + default: + break; + } + + if (modifiers_length || modifiers_sign) { + + switch (tok->kind) { + + case TOK_VOID: + case TOK__BOOL: + case TOK_FLOAT: + case TOK_STRUCT: + case TOK_UNION: + parse_error(tok, "invalid combination of types"); + return; + + case TOK_DOUBLE: + if (modifiers_sign != 0 || modifiers_length != 1) { + parse_error(tok, "invalid combination of types"); + return; + } + next_token(tok); + t1 = CTOP_LONGDOUBLE; + break; + + case TOK_CHAR: + if (modifiers_length != 0) { + parse_error(tok, "invalid combination of types"); + return; + } + modifiers_length = -2; + /* fall-through */ + case TOK_INT: + next_token(tok); + /* fall-through */ + default: + if (modifiers_sign >= 0) + switch (modifiers_length) { + case -2: t1 = CTOP_SCHAR; break; + case -1: t1 = CTOP_SHORT; break; + case 1: t1 = CTOP_LONG; break; + case 2: t1 = CTOP_LONGLONG; break; + default: t1 = CTOP_INT; break; + } + else + switch (modifiers_length) { + case -2: t1 = CTOP_UCHAR; break; + case -1: t1 = CTOP_USHORT; break; + case 1: t1 = CTOP_ULONG; break; + case 2: t1 = CTOP_ULONGLONG; break; + default: t1 = CTOP_UINT; break; + } + } + } + else { + switch (tok->kind) { + case TOK_INT: + t1 = CTOP_INT; + break; + case TOK_CHAR: + t1 = CTOP_CHAR; + break; + case TOK_VOID: + t1 = CTOP_VOID; + break; + case TOK__BOOL: + t1 = CTOP_BOOL; + break; + case TOK_FLOAT: + t1 = CTOP_FLOAT; + break; + case TOK_DOUBLE: + t1 = CTOP_DOUBLE; + break; + case TOK_IDENTIFIER: + { + abort(); +#if 0 + _crx_qual_type qt2; + char identifier[1024]; + if (tok->size >= 1024) { + parse_error(tok, "identifier name too long"); + return; + } + memcpy(identifier, tok->p, tok->size); + identifier[tok->size] = 0; + qt2 = tok->cb->get_user_type(tok->cb, identifier); + t1 = qt2.type; + result->qualifiers |= qt2.qualifiers; + break; +#endif + } + case TOK_STRUCT: + case TOK_UNION: + { + abort(); +#if 0 + char identifier[1024]; + int kind = tok->kind; + next_token(tok); + if (tok->kind != TOK_IDENTIFIER) { + parse_error(tok, "struct or union name expected"); + return; + } + if (tok->size >= 1024) { + parse_error(tok, "struct or union name too long"); + return; + } + memcpy(identifier, tok->p, tok->size); + identifier[tok->size] = 0; + if (kind == TOK_STRUCT) + t1 = tok->cb->get_struct_type(tok->cb, identifier); + else + t1 = tok->cb->get_union_type(tok->cb, identifier); + break; +#endif + } + default: + parse_error(tok, "identifier expected"); + return; + } + next_token(tok); + } + *alloc_ds(tok, 1) = t1; + if (const_qualifier) + *alloc_ds(tok, 1) = CTOP_CONST; + if (volatile_qualifier) + *alloc_ds(tok, 1) = CTOP_VOLATILE; + + //parse_sequel(tok, CTOP_END); + *alloc_ds(tok, 1) = CTOP_END; +} + + +int parse_c_type(const char *input, + ctype_opcode_t *output, size_t output_size, + const char **error_loc, const char **error_msg) +{ + token_t token; + + token.kind = TOK_START; + token.p = input; + token.error_location = error_loc; + token.error_message = error_msg; + token.size = 0; + token.opcodes = output; + token.opcodes_end = output + output_size; + next_token(&token); + parse_complete(&token); + + if (token.kind != TOK_END) { + parse_error(&token, "unexpected symbol"); + return -1; + } + return 0; +} diff --git a/new/parse_c_type.h b/new/parse_c_type.h new file mode 100644 --- /dev/null +++ b/new/parse_c_type.h @@ -0,0 +1,29 @@ + + +typedef int ctype_opcode_t; + +#define CTOP_END 0 +#define CTOP_CONST 1 +#define CTOP_VOLATILE 2 + +#define CTOP_VOID 100 +#define CTOP_BOOL 101 +#define CTOP_CHAR 102 +#define CTOP_SCHAR 103 +#define CTOP_UCHAR 104 +#define CTOP_SHORT 105 +#define CTOP_USHORT 106 +#define CTOP_INT 107 +#define CTOP_UINT 108 +#define CTOP_LONG 109 +#define CTOP_ULONG 110 +#define CTOP_LONGLONG 111 +#define CTOP_ULONGLONG 112 +#define CTOP_FLOAT 113 +#define CTOP_DOUBLE 114 +#define CTOP_LONGDOUBLE 115 + + +int parse_c_type(const char *input, + ctype_opcode_t *output, size_t output_size, + const char **error_loc, const char **error_msg); diff --git a/new/test_parse_c_type.py b/new/test_parse_c_type.py new file mode 100644 --- /dev/null +++ b/new/test_parse_c_type.py @@ -0,0 +1,53 @@ +import os +import cffi + +ffi = cffi.FFI() +ffi.cdef(""" +typedef int ctype_opcode_t; + +#define CTOP_END ... +#define CTOP_CONST ... +#define CTOP_VOLATILE ... + +#define CTOP_VOID ... +#define CTOP_BOOL ... +#define CTOP_CHAR ... +#define CTOP_SCHAR ... +#define CTOP_UCHAR ... +#define CTOP_SHORT ... +#define CTOP_USHORT ... +#define CTOP_INT ... +#define CTOP_UINT ... +#define CTOP_LONG ... +#define CTOP_ULONG ... +#define CTOP_LONGLONG ... +#define CTOP_ULONGLONG ... +#define CTOP_FLOAT ... +#define CTOP_DOUBLE ... +#define CTOP_LONGDOUBLE ... + +int parse_c_type(const char *input, + ctype_opcode_t *output, size_t output_size, + const char **error_loc, const char **error_msg); +""") + +lib = ffi.verify(open('parse_c_type.c').read(), + include_dirs=[os.getcwd()]) + + +def test_simple(): + out = ffi.new("ctype_opcode_t[]", 100) + for simple_type, expected in [ + ("int", lib.CTOP_INT), + ("signed int", lib.CTOP_INT), + (" long ", lib.CTOP_LONG), + ("long int", lib.CTOP_LONG), + ("unsigned short", lib.CTOP_USHORT), + ("long double", lib.CTOP_LONGDOUBLE), + ]: + for j in range(len(out)): + out[j] = -424242 + res = lib.parse_c_type(simple_type, out, 100, ffi.NULL, ffi.NULL) + assert res == 0 + assert out[0] == expected + assert out[1] == lib.CTOP_END From noreply at buildbot.pypy.org Wed Apr 8 19:41:58 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Wed, 8 Apr 2015 19:41:58 +0200 (CEST) Subject: [pypy-commit] pypy default: remove dead tests Message-ID: <20150408174158.AECAE1C1151@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76750:686fc31b6e07 Date: 2015-04-08 18:40 +0100 http://bitbucket.org/pypy/pypy/changeset/686fc31b6e07/ Log: remove dead tests diff --git a/rpython/rtyper/lltypesystem/test/test_lltype.py b/rpython/rtyper/lltypesystem/test/test_lltype.py --- a/rpython/rtyper/lltypesystem/test/test_lltype.py +++ b/rpython/rtyper/lltypesystem/test/test_lltype.py @@ -210,33 +210,6 @@ assert p1b == cast_pointer(Ptr(S1bis), p3) py.test.raises(RuntimeError, "cast_pointer(Ptr(S1), p3)") -def test_best_effort_gced_parent_detection(): - py.test.skip("test not relevant any more") - S2 = Struct("s2", ('a', Signed)) - S1 = GcStruct("s1", ('sub1', S2), ('sub2', S2), ('tail', Array(('e', Signed)))) - p1 = malloc(S1, 1) - p2 = p1.sub2 - p3 = p1.tail - p3[0].e = 1 - assert p3[0].e == 1 - del p1 - import gc - gc.collect() - py.test.raises(RuntimeError, "p2.a") - py.test.raises(RuntimeError, "p3[0]") - -def test_best_effort_gced_parent_for_arrays(): - py.test.skip("test not relevant any more") - A1 = GcArray(('v', Signed)) - p1 = malloc(A1, 10) - p1[5].v=3 - assert p1[5].v == 3 - p1_5 = p1[5] - del p1 - import gc - gc.collect() - py.test.raises(RuntimeError, "p1_5.v") - def test_examples(): A1 = GcArray(('v', Signed)) S = GcStruct("s", ('v', Signed)) @@ -388,11 +361,11 @@ s.x = 1 def type_info_S(p): return getRuntimeTypeInfo(S) - qp = functionptr(FuncType([Ptr(S)], Ptr(RuntimeTypeInfo)), - "type_info_S", + qp = functionptr(FuncType([Ptr(S)], Ptr(RuntimeTypeInfo)), + "type_info_S", _callable=type_info_S) - dp = functionptr(FuncType([Ptr(S)], Void), - "destructor_funcptr", + dp = functionptr(FuncType([Ptr(S)], Void), + "destructor_funcptr", _callable=f) pinf0 = attachRuntimeTypeInfo(S, qp, destrptr=dp) assert pinf0._obj.about == S @@ -422,8 +395,8 @@ return getRuntimeTypeInfo(S) else: return getRuntimeTypeInfo(S1) - fp = functionptr(FuncType([Ptr(S)], Ptr(RuntimeTypeInfo)), - "dynamic_type_info_S", + fp = functionptr(FuncType([Ptr(S)], Ptr(RuntimeTypeInfo)), + "dynamic_type_info_S", _callable=dynamic_type_info_S) attachRuntimeTypeInfo(S, fp) assert s.x == 0 @@ -434,7 +407,7 @@ py.test.raises(RuntimeError, "runtime_type_info(s1.sub)") s1.sub.x = 1 assert runtime_type_info(s1.sub) == getRuntimeTypeInfo(S1) - + def test_flavor_malloc(): def isweak(p, T): return p._weak and typeOf(p).TO == T @@ -450,7 +423,7 @@ p = malloc(T, flavor="gc") assert typeOf(p).TO == T assert not isweak(p, T) - + def test_opaque(): O = OpaqueType('O') p1 = opaqueptr(O, 'p1', hello="world") @@ -520,8 +493,8 @@ def test_adtmeths(): def h_newstruct(): return malloc(S) - - S = GcStruct('s', ('x', Signed), + + S = GcStruct('s', ('x', Signed), adtmeths={"h_newstruct": h_newstruct}) s = S.h_newstruct() @@ -553,15 +526,15 @@ def h_newstruct(S): return malloc(S) h_newstruct = typeMethod(h_newstruct) - - S = GcStruct('s', ('x', Signed), + + S = GcStruct('s', ('x', Signed), adtmeths={"h_newstruct": h_newstruct}) s = S.h_newstruct() assert typeOf(s) == Ptr(S) - Sprime = GcStruct('s', ('x', Signed), + Sprime = GcStruct('s', ('x', Signed), adtmeths={"h_newstruct": h_newstruct}) assert S == Sprime @@ -592,7 +565,7 @@ PA = Ptr(A) a = malloc(A, 2) assert cast_pointer(PA, a) == a - + def test_array_with_no_length(): A = GcArray(Signed, hints={'nolength': True}) a = malloc(A, 10) @@ -604,7 +577,7 @@ s = malloc(GcS) s.x = 1 assert list(dissect_ll_instance(s)) == [(Ptr(GcS), s), (GcS, s._obj), (Signed, 1)] - + A = GcArray(('x', Signed)) a = malloc(A, 10) for i in range(10): From noreply at buildbot.pypy.org Thu Apr 9 11:36:37 2015 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 9 Apr 2015 11:36:37 +0200 (CEST) Subject: [pypy-commit] benchmarks single-run: oops Message-ID: <20150409093637.BF14C1C1EED@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: single-run Changeset: r325:97737211ed68 Date: 2015-04-09 11:36 +0200 http://bitbucket.org/pypy/benchmarks/changeset/97737211ed68/ Log: oops diff --git a/benchmarks.py b/benchmarks.py --- a/benchmarks.py +++ b/benchmarks.py @@ -62,8 +62,7 @@ 'eparse' : {'bm_env': {'PYTHONPATH': relative('lib/monte')}}, 'bm_mako' : {'bm_env': {'PYTHONPATH': relative('lib/mako')}}, 'bm_dulwich_log': {'bm_env': {'PYTHONPATH': relative('lib/dulwich-0.9.1')}}, - 'bm_chameleon': {'bm_env': {'PYTHONPATH': relative('lib/chameleon/src')}, - 'iteration_scaling': 3}, + 'bm_chameleon': {'bm_env': {'PYTHONPATH': relative('lib/chameleon/src')}}, } for name in ['expand', 'integrate', 'sum', 'str']: From noreply at buildbot.pypy.org Thu Apr 9 12:41:03 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 9 Apr 2015 12:41:03 +0200 (CEST) Subject: [pypy-commit] stmgc c8-locking: A branch to check and change the locks Message-ID: <20150409104103.059FC1C010B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-locking Changeset: r1747:fcb13e931f85 Date: 2015-04-09 12:09 +0200 http://bitbucket.org/pypy/stmgc/changeset/fcb13e931f85/ Log: A branch to check and change the locks From noreply at buildbot.pypy.org Thu Apr 9 12:41:04 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 9 Apr 2015 12:41:04 +0200 (CEST) Subject: [pypy-commit] stmgc c8-locking: A summary of the locks used in c8. Message-ID: <20150409104104.2CCE41C010B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-locking Changeset: r1748:fe2ead99a2d3 Date: 2015-04-09 12:41 +0200 http://bitbucket.org/pypy/stmgc/changeset/fe2ead99a2d3/ Log: A summary of the locks used in c8. diff --git a/c8/LOCKS b/c8/LOCKS new file mode 100644 --- /dev/null +++ b/c8/LOCKS @@ -0,0 +1,55 @@ + + +main lock-free operation +======================== + +The main lock-free operation is at commit time: the compare-and-swap +that attaches a new log entry after 'last_commit_log_entry'. + + + +modification_lock +================= + +one per segment. + +acquired on segment N when we want to read or write the segment N's +copy of 'modified_old_objects', the backup copies, etc. + +an important user is _stm_validate(): it locks the current segment, +and all the other segments out of which it is going to read data + +could be improved, because _stm_validate() writes into the current +segment but only reads the other ones. So far it mostly serializes +calls to _stm_validate(): if we have two of them starting at roughly +the same time, they need both to acquire the modification_lock of at +least the segment that did the most recent commit --- even though it +could proceed in parallel if they could both realize that they only +want to read from that same segment. + +same, handle_segfault_in_page() acquires two modification_locks: the +current segment (which needs to be written to), and the +'copy_from_segnum' (which only needs to be read from). + +the current segment modification_lock is also acquired briefly +whenever we change our segment's 'modified_old_objects'. + + + +privatization_lock +================== + +one per segment. Works like a single "reader-writer" lock: each +segment acquires either only its copy ("reader") or all of them +("writer"). + +"Reader" status is needed to call get_page_status_in(). +"Writer" status is needed to call set_page_status_in/page_mark_(in)accessible. + +Essential "writers": +- handle_segfault_in_page(), but it only writes the status for the current seg + +Essential "readers": +- _stm_validate() +- push_large_overflow_objects_to_other_segments() +- nursery.c calling synchronize_object_enqueue() diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -871,6 +871,8 @@ static void touch_all_pages_of_obj(object_t *obj, size_t obj_size) { + /* XXX should it be simpler, just really trying to read a dummy + byte in each page? */ int my_segnum = STM_SEGMENT->segment_num; uintptr_t end_page, first_page = ((uintptr_t)obj) / 4096UL; From noreply at buildbot.pypy.org Thu Apr 9 12:55:33 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Thu, 9 Apr 2015 12:55:33 +0200 (CEST) Subject: [pypy-commit] pypy default: STORE_SUBSCR pops three things from the stack, not two Message-ID: <20150409105533.79D191C010B@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: Changeset: r76751:0c2d248dc5ec Date: 2015-04-09 12:55 +0200 http://bitbucket.org/pypy/pypy/changeset/0c2d248dc5ec/ Log: STORE_SUBSCR pops three things from the stack, not two diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py --- a/pypy/interpreter/astcompiler/assemble.py +++ b/pypy/interpreter/astcompiler/assemble.py @@ -576,7 +576,7 @@ ops.DELETE_SLICE+2: -2, ops.DELETE_SLICE+3: -3, - ops.STORE_SUBSCR: -2, + ops.STORE_SUBSCR: -3, ops.DELETE_SUBSCR: -2, ops.GET_ITER: 0, From noreply at buildbot.pypy.org Thu Apr 9 13:07:04 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 9 Apr 2015 13:07:04 +0200 (CEST) Subject: [pypy-commit] stmgc c8-locking: more doc Message-ID: <20150409110704.22EFD1C01F3@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-locking Changeset: r1749:d1eed2e7d2a4 Date: 2015-04-09 13:07 +0200 http://bitbucket.org/pypy/stmgc/changeset/d1eed2e7d2a4/ Log: more doc diff --git a/c8/LOCKS b/c8/LOCKS --- a/c8/LOCKS +++ b/c8/LOCKS @@ -53,3 +53,19 @@ - _stm_validate() - push_large_overflow_objects_to_other_segments() - nursery.c calling synchronize_object_enqueue() + + + +mutex and conditions +==================== + +There is also one global mutex and a few condition codes. It's +unclear if these are still the best solution. + +The mutex is acquired in stm_start_transaction() and in +stm_commit_transaction(). The main purpose is to wait for or signal +the C_SEGMENT_FREE condition code. + +The C_AT_SAFE_POINT and C_REQUEST_REMOVED condition codes are used by +synchronize_all_threads(). That's used only in rare cases, for +example because we want to start a major collection. From noreply at buildbot.pypy.org Thu Apr 9 14:02:27 2015 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 9 Apr 2015 14:02:27 +0200 (CEST) Subject: [pypy-commit] pypy dtrace-support: merge default Message-ID: <20150409120227.1AA4F1C0173@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: dtrace-support Changeset: r76752:e7893a492224 Date: 2015-04-09 12:11 +0200 http://bitbucket.org/pypy/pypy/changeset/e7893a492224/ Log: merge default diff too long, truncating to 2000 out of 14913 lines diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -3,11 +3,10 @@ d8ac7d23d3ec5f9a0fa1264972f74a010dbfd07f release-1.6 ff4af8f318821f7f5ca998613a60fca09aa137da release-1.7 07e08e9c885ca67d89bcc304e45a32346daea2fa release-2.0-beta-1 -9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm -9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm 20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0 394146e9bb673514c61f0150ab2013ccf78e8de7 release-2.3 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1 10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0 +9c4588d731b7fe0b08669bd732c2b676cb0a8233 release-2.5.1 diff --git a/.tddium.requirements.txt b/.tddium.requirements.txt deleted file mode 100644 --- a/.tddium.requirements.txt +++ /dev/null @@ -1,1 +0,0 @@ -pytest diff --git a/lib-python/2.7/test/test_urllib2net.py b/lib-python/2.7/test/test_urllib2net.py --- a/lib-python/2.7/test/test_urllib2net.py +++ b/lib-python/2.7/test/test_urllib2net.py @@ -102,11 +102,8 @@ def test_ftp(self): urls = [ - 'ftp://ftp.kernel.org/pub/linux/kernel/README', - 'ftp://ftp.kernel.org/pub/linux/kernel/non-existent-file', - #'ftp://ftp.kernel.org/pub/leenox/kernel/test', - 'ftp://gatekeeper.research.compaq.com/pub/DEC/SRC' - '/research-reports/00README-Legal-Rules-Regs', + 'ftp://ftp.debian.org/debian/README', + 'ftp://ftp.debian.org/debian/non-existent-file', ] self._test_urls(urls, self._extra_handlers()) @@ -255,6 +252,7 @@ with test_support.transient_internet(url, timeout=None): u = _urlopen_with_retry(url) self.assertIsNone(u.fp._sock.fp._sock.gettimeout()) + u.close() def test_http_default_timeout(self): self.assertIsNone(socket.getdefaulttimeout()) @@ -266,6 +264,7 @@ finally: socket.setdefaulttimeout(None) self.assertEqual(u.fp._sock.fp._sock.gettimeout(), 60) + u.close() def test_http_no_timeout(self): self.assertIsNone(socket.getdefaulttimeout()) @@ -277,20 +276,23 @@ finally: socket.setdefaulttimeout(None) self.assertIsNone(u.fp._sock.fp._sock.gettimeout()) + u.close() def test_http_timeout(self): url = "http://www.example.com" with test_support.transient_internet(url): u = _urlopen_with_retry(url, timeout=120) self.assertEqual(u.fp._sock.fp._sock.gettimeout(), 120) + u.close() - FTP_HOST = "ftp://ftp.mirror.nl/pub/gnu/" + FTP_HOST = 'ftp://ftp.debian.org/debian/' def test_ftp_basic(self): self.assertIsNone(socket.getdefaulttimeout()) with test_support.transient_internet(self.FTP_HOST, timeout=None): u = _urlopen_with_retry(self.FTP_HOST) self.assertIsNone(u.fp.fp._sock.gettimeout()) + u.close() def test_ftp_default_timeout(self): self.assertIsNone(socket.getdefaulttimeout()) @@ -301,6 +303,7 @@ finally: socket.setdefaulttimeout(None) self.assertEqual(u.fp.fp._sock.gettimeout(), 60) + u.close() def test_ftp_no_timeout(self): self.assertIsNone(socket.getdefaulttimeout(),) @@ -311,11 +314,16 @@ finally: socket.setdefaulttimeout(None) self.assertIsNone(u.fp.fp._sock.gettimeout()) + u.close() def test_ftp_timeout(self): with test_support.transient_internet(self.FTP_HOST): - u = _urlopen_with_retry(self.FTP_HOST, timeout=60) + try: + u = _urlopen_with_retry(self.FTP_HOST, timeout=60) + except: + raise self.assertEqual(u.fp.fp._sock.gettimeout(), 60) + u.close() def test_main(): diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -454,6 +454,7 @@ self.__cursors_counter = 0 self.__statements = [] self.__statements_counter = 0 + self.__rawstatements = set() self._statement_cache = _StatementCache(self, cached_statements) self.__func_cache = {} @@ -483,6 +484,14 @@ self.__do_all_statements(Statement._finalize, True) + # depending on when this close() is called, the statements' weakrefs + # may be already dead, even though Statement.__del__() was not called + # yet. In this case, self.__rawstatements is not empty. + if self.__rawstatements is not None: + for stmt in list(self.__rawstatements): + self._finalize_raw_statement(stmt) + self.__rawstatements = None + if self._db: ret = _lib.sqlite3_close(self._db) if ret != _lib.SQLITE_OK: @@ -562,6 +571,7 @@ self.__cursors = [r for r in self.__cursors if r() is not None] def _remember_statement(self, statement): + self.__rawstatements.add(statement._statement) self.__statements.append(weakref.ref(statement)) self.__statements_counter += 1 if self.__statements_counter < 200: @@ -569,6 +579,11 @@ self.__statements_counter = 0 self.__statements = [r for r in self.__statements if r() is not None] + def _finalize_raw_statement(self, _statement): + if self.__rawstatements is not None: + self.__rawstatements.remove(_statement) + _lib.sqlite3_finalize(_statement) + def __do_all_statements(self, action, reset_cursors): for weakref in self.__statements: statement = weakref() @@ -1199,7 +1214,6 @@ def __init__(self, connection, sql): self.__con = connection - self.__con._remember_statement(self) self._in_use = False @@ -1244,17 +1258,19 @@ if ret != _lib.SQLITE_OK: raise self.__con._get_exception(ret) + self.__con._remember_statement(self) + tail = _ffi.string(next_char[0]).decode('utf-8') if _check_remaining_sql(tail): raise Warning("You can only execute one statement at a time.") def __del__(self): if self._statement: - _lib.sqlite3_finalize(self._statement) + self.__con._finalize_raw_statement(self._statement) def _finalize(self): if self._statement: - _lib.sqlite3_finalize(self._statement) + self.__con._finalize_raw_statement(self._statement) self._statement = None self._in_use = False diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py --- a/lib_pypy/_tkinter/app.py +++ b/lib_pypy/_tkinter/app.py @@ -96,7 +96,7 @@ if not self.threaded: # TCL is not thread-safe, calls needs to be serialized. - self._tcl_lock = threading.Lock() + self._tcl_lock = threading.RLock() else: self._tcl_lock = _DummyLock() diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py --- a/lib_pypy/_tkinter/tklib.py +++ b/lib_pypy/_tkinter/tklib.py @@ -1,7 +1,7 @@ # C bindings with libtcl and libtk. from cffi import FFI -import sys +import sys, os tkffi = FFI() @@ -135,9 +135,12 @@ linklibs = ['tcl', 'tk'] libdirs = [] else: - incdirs=['/usr/include/tcl'] - linklibs=['tcl', 'tk'] - libdirs = [] + for _ver in ['', '8.6', '8.5', '']: + incdirs = ['/usr/include/tcl' + _ver] + linklibs = ['tcl' + _ver, 'tk' + _ver] + libdirs = [] + if os.path.isdir(incdirs[0]): + break tklib = tkffi.verify(""" #include diff --git a/lib_pypy/cffi.egg-info b/lib_pypy/cffi.egg-info --- a/lib_pypy/cffi.egg-info +++ b/lib_pypy/cffi.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: cffi -Version: 0.8.6+ +Version: 0.9.2 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "0.8.6+" -__version_info__ = (0, 8, 6, "plus") +__version__ = "0.9.2" +__version_info__ = (0, 9, 2) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/gdbm.py b/lib_pypy/gdbm.py --- a/lib_pypy/gdbm.py +++ b/lib_pypy/gdbm.py @@ -20,9 +20,11 @@ } datum; datum gdbm_fetch(void*, datum); +datum pygdbm_fetch(void*, char*, int); int gdbm_delete(void*, datum); int gdbm_store(void*, datum, datum, int); int gdbm_exists(void*, datum); +int pygdbm_exists(void*, char*, int); int gdbm_reorganize(void*); @@ -37,19 +39,29 @@ ''') try: + verify_code = ''' + #include "gdbm.h" + + static datum pygdbm_fetch(GDBM_FILE gdbm_file, char *dptr, int dsize) { + datum key = {dptr, dsize}; + return gdbm_fetch(gdbm_file, key); + } + + static int pygdbm_exists(GDBM_FILE gdbm_file, char *dptr, int dsize) { + datum key = {dptr, dsize}; + return gdbm_exists(gdbm_file, key); + } + + ''' if sys.platform.startswith('freebsd'): import os.path _localbase = os.environ.get('LOCALBASE', '/usr/local') - lib = ffi.verify(''' - #include "gdbm.h" - ''', libraries=['gdbm'], + lib = ffi.verify(verify_code, libraries=['gdbm'], include_dirs=[os.path.join(_localbase, 'include')], library_dirs=[os.path.join(_localbase, 'lib')] ) else: - lib = ffi.verify(''' - #include "gdbm.h" - ''', libraries=['gdbm']) + lib = ffi.verify(verify_code, libraries=['gdbm']) except cffi.VerificationError as e: # distutils does not preserve the actual message, # but the verification is simple enough that the @@ -59,6 +71,13 @@ class error(Exception): pass +def _checkstr(key): + if isinstance(key, unicode): + key = key.encode("ascii") + if not isinstance(key, str): + raise TypeError("gdbm mappings have string indices only") + return key + def _fromstr(key): if isinstance(key, unicode): key = key.encode("ascii") @@ -107,12 +126,14 @@ def __contains__(self, key): self._check_closed() - return lib.gdbm_exists(self.ll_dbm, _fromstr(key)) + key = _checkstr(key) + return lib.pygdbm_exists(self.ll_dbm, key, len(key)) has_key = __contains__ def __getitem__(self, key): self._check_closed() - drec = lib.gdbm_fetch(self.ll_dbm, _fromstr(key)) + key = _checkstr(key) + drec = lib.pygdbm_fetch(self.ll_dbm, key, len(key)) if not drec.dptr: raise KeyError(key) res = str(ffi.buffer(drec.dptr, drec.dsize)) diff --git a/lib_pypy/pyrepl/readline.py b/lib_pypy/pyrepl/readline.py --- a/lib_pypy/pyrepl/readline.py +++ b/lib_pypy/pyrepl/readline.py @@ -73,7 +73,6 @@ assume_immutable_completions = False use_brackets = False sort_in_column = True - tab_insert_spaces_if_stem_is_empty = False def error(self, msg="none"): pass # don't show error messages by default @@ -87,7 +86,7 @@ return ''.join(b[p+1:self.pos]) def get_completions(self, stem): - if len(stem) == 0 and self.tab_insert_spaces_if_stem_is_empty: + if len(stem) == 0 and self.more_lines is not None: b = self.buffer p = self.pos while p > 0 and b[p - 1] != '\n': @@ -141,12 +140,16 @@ def collect_keymap(self): return super(ReadlineAlikeReader, self).collect_keymap() + ( - (r'\n', 'maybe-accept'),) + (r'\n', 'maybe-accept'), + (r'\', 'backspace-dedent'), + ) def __init__(self, console): super(ReadlineAlikeReader, self).__init__(console) self.commands['maybe_accept'] = maybe_accept self.commands['maybe-accept'] = maybe_accept + self.commands['backspace_dedent'] = backspace_dedent + self.commands['backspace-dedent'] = backspace_dedent def after_command(self, cmd): super(ReadlineAlikeReader, self).after_command(cmd) @@ -164,6 +167,28 @@ if self.pos > len(self.buffer): self.pos = len(self.buffer) +def _get_this_line_indent(buffer, pos): + indent = 0 + while pos > 0 and buffer[pos - 1] in " \t": + indent += 1 + pos -= 1 + if pos > 0 and buffer[pos - 1] == "\n": + return indent + return 0 + +def _get_previous_line_indent(buffer, pos): + prevlinestart = pos + while prevlinestart > 0 and buffer[prevlinestart - 1] != "\n": + prevlinestart -= 1 + prevlinetext = prevlinestart + while prevlinetext < pos and buffer[prevlinetext] in " \t": + prevlinetext += 1 + if prevlinetext == pos: + indent = None + else: + indent = prevlinetext - prevlinestart + return prevlinestart, indent + class maybe_accept(commands.Command): def do(self): r = self.reader @@ -172,13 +197,39 @@ # if there are already several lines and the cursor # is not on the last one, always insert a new \n. text = r.get_unicode() - if "\n" in r.buffer[r.pos:]: + if ("\n" in r.buffer[r.pos:] or + (r.more_lines is not None and r.more_lines(text))): + # + # auto-indent the next line like the previous line + prevlinestart, indent = _get_previous_line_indent(r.buffer, r.pos) r.insert("\n") - elif r.more_lines is not None and r.more_lines(text): - r.insert("\n") + if indent: + for i in range(prevlinestart, prevlinestart + indent): + r.insert(r.buffer[i]) else: self.finish = 1 +class backspace_dedent(commands.Command): + def do(self): + r = self.reader + b = r.buffer + if r.pos > 0: + repeat = 1 + if b[r.pos - 1] != "\n": + indent = _get_this_line_indent(b, r.pos) + if indent > 0: + ls = r.pos - indent + while ls > 0: + ls, pi = _get_previous_line_indent(b, ls - 1) + if pi is not None and pi < indent: + repeat = indent - pi + break + r.pos -= repeat + del b[r.pos:r.pos + repeat] + r.dirty = 1 + else: + self.reader.error("can't backspace at start") + # ____________________________________________________________ class _ReadlineWrapper(object): @@ -212,15 +263,14 @@ boolean value is true. """ reader = self.get_reader() - saved = reader.more_lines, reader.tab_insert_spaces_if_stem_is_empty + saved = reader.more_lines try: reader.more_lines = more_lines reader.ps1 = reader.ps2 = ps1 reader.ps3 = reader.ps4 = ps2 - reader.tab_insert_spaces_if_stem_is_empty = True return reader.readline(returns_unicode=returns_unicode) finally: - reader.more_lines, reader.tab_insert_spaces_if_stem_is_empty = saved + reader.more_lines = saved def parse_and_bind(self, string): pass # XXX we don't support parsing GNU-readline-style init files diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -36,7 +36,7 @@ def pytest_addoption(parser): from rpython.conftest import pytest_addoption pytest_addoption(parser) - + group = parser.getgroup("pypy options") group.addoption('-A', '--runappdirect', action="store_true", default=False, dest="runappdirect", @@ -44,6 +44,9 @@ group.addoption('--direct', action="store_true", default=False, dest="rundirect", help="run pexpect tests directly") + group.addoption('--raise-operr', action="store_true", + default=False, dest="raise_operr", + help="Show the interp-level OperationError in app-level tests") def pytest_funcarg__space(request): from pypy.tool.pytest.objspace import gettestobjspace diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst --- a/pypy/doc/build.rst +++ b/pypy/doc/build.rst @@ -146,6 +146,26 @@ :doc:`objspace proxies ` document. +Packaging (preparing for installation) +-------------------------------------- + +Packaging is required if you want to install PyPy system-wide, even to +install on the same machine. The reason is that doing so prepares a +number of extra features that cannot be done lazily on a root-installed +PyPy, because the normal users don't have write access. This concerns +mostly libraries that would normally be compiled if and when they are +imported the first time. + +:: + + cd pypy/tool/release + ./package.py pypy-VER-PLATFORM + +This creates a clean and prepared hierarchy, as well as a ``.tar.bz2`` +with the same content; both are found by default in +``/tmp/usession-YOURNAME/build/``. You can then either move the file +hierarchy or unpack the ``.tar.bz2`` at the correct place. + Installation ------------ diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -58,7 +58,7 @@ # General information about the project. project = u'PyPy' -copyright = u'2014, The PyPy Project' +copyright = u'2015, The PyPy Project' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -67,7 +67,7 @@ # The short X.Y version. version = '2.5' # The full version, including alpha/beta/rc tags. -release = '2.5.0' +release = '2.5.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst --- a/pypy/doc/embedding.rst +++ b/pypy/doc/embedding.rst @@ -36,7 +36,8 @@ "PyPy home directory". The arguments are: * ``home``: NULL terminated path to an executable inside the pypy directory - (can be a .so name, can be made up) + (can be a .so name, can be made up). Used to look up the standard + library, and is also set as ``sys.executable``. * ``verbose``: if non-zero, it will print error messages to stderr diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-2.5.1.rst release-2.5.0.rst release-2.4.0.rst release-2.3.1.rst diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,7 @@ .. toctree:: whatsnew-head.rst + whatsnew-2.5.1.rst whatsnew-2.5.0.rst whatsnew-2.4.0.rst whatsnew-2.3.1.rst diff --git a/pypy/doc/jit-hooks.rst b/pypy/doc/jit-hooks.rst --- a/pypy/doc/jit-hooks.rst +++ b/pypy/doc/jit-hooks.rst @@ -39,3 +39,30 @@ Reason is a string, the meaning of other arguments is the same as attributes on JitLoopInfo object +.. function:: enable_debug() + + Start recording debugging counters for ``get_stats_snapshot`` + +.. function:: disable_debug() + + Stop recording debugging counters for ``get_stats_snapshot`` + +.. function:: get_stats_snapshot() + + Get the jit status in the specific moment in time. Note that this + is eager - the attribute access is not lazy, if you need new stats + you need to call this function again. You might want to call + ``enable_debug`` to get more information. It returns an instance + of ``JitInfoSnapshot`` + +.. class:: JitInfoSnapshot + + A class describing current snapshot. Usable attributes: + + * ``counters`` - internal JIT integer counters + + * ``counter_times`` - internal JIT float counters, notably time spent + TRACING and in the JIT BACKEND + + * ``loop_run_times`` - counters for number of times loops are run, only + works when ``enable_debug`` is called. diff --git a/pypy/doc/release-2.5.1.rst b/pypy/doc/release-2.5.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-2.5.1.rst @@ -0,0 +1,115 @@ +================================ +PyPy 2.5.1 - Pineapple Bromeliad +================================ + +We're pleased to announce PyPy 2.5.1, Pineapple `Bromeliad`_ following on the heels of 2.5.0 + +You can download the PyPy 2.5.1 release here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project, and for those who donate to our three sub-projects, as well as our +volunteers and contributors. +We've shown quite a bit of progress, but we're slowly running out of funds. +Please consider donating more, or even better convince your employer to donate, +so we can finish those projects! The three sub-projects are: + +* `Py3k`_ (supporting Python 3.x): We have released a Python 3.2.5 compatible version + we call PyPy3 2.4.0, and are working toward a Python 3.3 compatible version + +* `STM`_ (software transactional memory): We have released a first working version, + and continue to try out new promising paths of achieving a fast multithreaded Python + +* `NumPy`_ which requires installation of our fork of upstream numpy, + available `on bitbucket`_ + +.. _`Bromeliad`: http://xkcd.com/1498 +.. _`Py3k`: http://pypy.org/py3donate.html +.. _`STM`: http://pypy.org/tmdonate2.html +.. _`NumPy`: http://pypy.org/numpydonate.html +.. _`on bitbucket`: https://www.bitbucket.org/pypy/numpy + +We would also like to encourage new people to join the project. PyPy has many +layers and we need help with all of them: `PyPy`_ and `Rpython`_ documentation +improvements, tweaking popular `modules`_ to run on pypy, or general `help`_ with making +Rpython's JIT even better. + +.. _`PyPy`: http://doc.pypy.org +.. _`Rpython`: http://rpython.readthedocs.org +.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7. It's fast (`pypy and cpython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +This release supports **x86** machines on most common operating systems +(Linux 32/64, Mac OS X 64, Windows, and OpenBSD), +as well as newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux. + +While we support 32 bit python on Windows, work on the native Windows 64 +bit python is still stalling, we would welcome a volunteer +to `handle that`_. + +.. _`pypy and cpython 2.7.x`: http://speed.pypy.org +.. _`handle that`: http://doc.pypy.org/en/latest/windows.html#what-is-missing-for-a-full-64-bit-translation + +Highlights +========== + +* The past months have seen pypy mature and grow, as rpython becomes the goto + solution for writing fast dynamic language interpreters. Our separation of + Rpython and the python interpreter PyPy is now much clearer in the + `PyPy documentation`_ and we now have seperate `RPython documentation`_. + Tell us what still isn't clear, or even better help us improve the documentation. + +* We merged version 2.7.9 of python's stdlib. From the python release notice: + + * The entirety of Python 3.4's `ssl module`_ has been backported. + See `PEP 466`_ for justification. + + * HTTPS certificate validation using the system's certificate store is now + enabled by default. See `PEP 476`_ for details. + + * SSLv3 has been disabled by default in httplib and its reverse dependencies + due to the `POODLE attack`_. + + * The `ensurepip module`_ has been backported, which provides the pip + package manager in every Python 2.7 installation. See `PEP 477`_. + +* The garbage collector now ignores parts of the stack which did not change + since the last collection, another performance boost + +* errno and LastError are saved around cffi calls so things like pdb will not + overwrite it + +* We continue to asymptotically approach a score of 7 times faster than cpython + on our benchmark suite, we now rank 6.98 on latest runs + +* Issues reported with our previous release were resolved_ after reports from users on + our issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at + #pypy. + +.. _`PyPy documentation`: http://doc.pypy.org +.. _`RPython documentation`: http://rpython.readthedocs.org +.. _`ssl module`: https://docs.python.org/3/library/ssl.html +.. _`PEP 466`: https://www.python.org/dev/peps/pep-0466 +.. _`PEP 476`: https://www.python.org/dev/peps/pep-0476 +.. _`PEP 477`: https://www.python.org/dev/peps/pep-0477 +.. _`POODLE attack`: https://www.imperialviolet.org/2014/10/14/poodle.html +.. _`ensurepip module`: https://docs.python.org/2/library/ensurepip.html +.. _resolved: http://doc.pypy.org/en/latest/whatsnew-2.5.1.html + +Please try it out and let us know what you think. We welcome +success stories, `experiments`_, or `benchmarks`_, we know you are using PyPy, please tell us about it! + +Cheers + +The PyPy Team + +.. _`experiments`: http://morepypy.blogspot.com/2015/02/experiments-in-pyrlang-with-rpython.html +.. _`benchmarks`: https://mithrandi.net/blog/2015/03/axiom-benchmark-results-on-pypy-2-5-0 diff --git a/pypy/doc/stm.rst b/pypy/doc/stm.rst --- a/pypy/doc/stm.rst +++ b/pypy/doc/stm.rst @@ -25,11 +25,12 @@ .. _`2nd call for donation`: http://pypy.org/tmdonate2.html -Introduction -============ +What pypy-stm is for +==================== ``pypy-stm`` is a variant of the regular PyPy interpreter. (This -version supports Python 2.7; see below for `Python 3`_.) With caveats_ +version supports Python 2.7; see below for `Python 3, CPython, +and others`_.) With caveats_ listed below, it should be in theory within 20%-50% slower than a regular PyPy, comparing the JIT version in both cases (but see below!). It is called @@ -45,15 +46,36 @@ it as a drop-in replacement and multithreaded programs will run on multiple cores. -* ``pypy-stm`` does not impose any special API to the user, but it - provides a new pure Python module called `transactional_memory`_ with - features to inspect the state or debug conflicts_ that prevent - parallelization. This module can also be imported on top of a non-STM - PyPy or CPython. +* ``pypy-stm`` provides (but does not impose) a special API to the + user in the pure Python module ``transaction``. This module is based + on the lower-level module ``pypystm``, but also provides some + compatibily with non-STM PyPy's or CPython's. * Building on top of the way the GIL is removed, we will talk - about `Atomic sections, Transactions, etc.: a better way to write - parallel programs`_. + about `How to write multithreaded programs: the 10'000-feet view`_ + and `transaction.TransactionQueue`_. + + +...and what pypy-stm is not for +------------------------------- + +``pypy-stm`` gives a Python without the GIL. This means that it is +useful in situations where the GIL is the problem in the first place. +(This includes cases where the program can easily be modified to run +in multiple threads; often, we don't consider doing that precisely +because of the GIL.) + +However, there are plenty of cases where the GIL is not the problem. +Do not hope ``pypy-stm`` to be helpful in these cases! This includes +all programs that use multiple threads but don't actually spend a lot +of time running Python code. For example, it may be spending all its +time waiting for I/O to occur, or performing some long computation on +a huge matrix. These are cases where the CPU is either idle, or in +some C/Fortran library anyway; in both cases, the interpreter (either +CPython or the regular PyPy) should release the GIL around the +external calls. The threads will thus not end up fighting for the +GIL. + Getting Started @@ -63,9 +85,10 @@ Development is done in the branch `stmgc-c7`_. If you are only interested in trying it out, you can download a Ubuntu binary here__ -(``pypy-stm-2.3*.tar.bz2``, Ubuntu 12.04-14.04). The current version +(``pypy-stm-2.*.tar.bz2``, for Ubuntu 12.04-14.04). The current version supports four "segments", which means that it will run up to four -threads in parallel. +threads in parallel. (Development recently switched to `stmgc-c8`_, +but that is not ready for trying out yet.) To build a version from sources, you first need to compile a custom version of clang(!); we recommend downloading `llvm and clang like @@ -78,6 +101,7 @@ rpython/bin/rpython -Ojit --stm pypy/goal/targetpypystandalone.py .. _`stmgc-c7`: https://bitbucket.org/pypy/pypy/src/stmgc-c7/ +.. _`stmgc-c8`: https://bitbucket.org/pypy/pypy/src/stmgc-c8/ .. __: https://bitbucket.org/pypy/pypy/downloads/ .. __: http://clang.llvm.org/get_started.html .. __: https://bitbucket.org/pypy/stmgc/src/default/c7/llvmfix/ @@ -85,60 +109,78 @@ .. _caveats: -Current status --------------- +Current status (stmgc-c7) +------------------------- -* So far, small examples work fine, but there are still a few bugs. - We're busy fixing them as we find them; feel free to `report bugs`_. +* **NEW:** It seems to work fine, without crashing any more. Please `report + any crash`_ you find (or other bugs). * It runs with an overhead as low as 20% on examples like "richards". There are also other examples with higher overheads --currently up to 2x for "translate.py"-- which we are still trying to understand. One suspect is our partial GC implementation, see below. +* **NEW:** the ``PYPYSTM`` environment variable and the + ``pypy/stm/print_stm_log.py`` script let you know exactly which + "conflicts" occurred. This is described in the section + `transaction.TransactionQueue`_ below. + +* **NEW:** special transaction-friendly APIs (like ``stmdict``), + described in the section `transaction.TransactionQueue`_ below. The + old API changed again, mostly moving to different modules. Sorry + about that. I feel it's a better idea to change the API early + instead of being stuck with a bad one later... + * Currently limited to 1.5 GB of RAM (this is just a parameter in - `core.h`__). Memory overflows are not correctly handled; they cause - segfaults. + `core.h`__ -- theoretically. In practice, increase it too much and + clang crashes again). Memory overflows are not correctly handled; + they cause segfaults. -* The JIT warm-up time improved recently but is still bad. In order to - produce machine code, the JIT needs to enter a special single-threaded - mode for now. This means that you will get bad performance results if - your program doesn't run for several seconds, where *several* can mean - *many.* When trying benchmarks, be sure to check that you have - reached the warmed state, i.e. the performance is not improving any - more. This should be clear from the fact that as long as it's - producing more machine code, ``pypy-stm`` will run on a single core. +* **NEW:** The JIT warm-up time improved again, but is still + relatively large. In order to produce machine code, the JIT needs + to enter "inevitable" mode. This means that you will get bad + performance results if your program doesn't run for several seconds, + where *several* can mean *many.* When trying benchmarks, be sure to + check that you have reached the warmed state, i.e. the performance + is not improving any more. * The GC is new; although clearly inspired by PyPy's regular GC, it misses a number of optimizations for now. Programs allocating large numbers of small objects that don't immediately die (surely a common - situation) suffer from these missing optimizations. + situation) suffer from these missing optimizations. (The bleeding + edge ``stmgc-c8`` is better at that.) -* The GC has no support for destructors: the ``__del__`` method is never - called (including on file objects, which won't be closed for you). - This is of course temporary. Also, weakrefs might appear to work a - bit strangely for now (staying alive even though ``gc.collect()``, or - even dying but then un-dying for a short time before dying again). +* Weakrefs might appear to work a bit strangely for now, sometimes + staying alive throught ``gc.collect()``, or even dying but then + un-dying for a short time before dying again. A similar problem can + show up occasionally elsewhere with accesses to some external + resources, where the (apparent) serialized order doesn't match the + underlying (multithreading) order. These are bugs (partially fixed + already in ``stmgc-c8``). Also, debugging helpers like + ``weakref.getweakrefcount()`` might give wrong answers. * The STM system is based on very efficient read/write barriers, which are mostly done (their placement could be improved a bit in - JIT-generated machine code). But the overall bookkeeping logic could - see more improvements (see `Low-level statistics`_ below). + JIT-generated machine code). * Forking the process is slow because the complete memory needs to be copied manually. A warning is printed to this effect. * Very long-running processes (on the order of days) will eventually crash on an assertion error because of a non-implemented overflow of - an internal 29-bit number. + an internal 28-bit counter. -.. _`report bugs`: https://bugs.pypy.org/ +* The recursion detection code was not reimplemented. Infinite + recursion just segfaults for now. + + +.. _`report any crash`: https://bitbucket.org/pypy/pypy/issues?status=new&status=open .. __: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/rpython/translator/stm/src_stm/stm/core.h -Python 3 -======== +Python 3, CPython, and others +============================= In this document I describe "pypy-stm", which is based on PyPy's Python 2.7 interpreter. Supporting Python 3 should take about half an @@ -153,12 +195,66 @@ framework, although the amount of work to put there might vary, because the STM framework within RPython is currently targeting the PyPy interpreter and other ones might have slightly different needs. +But in general, all the tedious transformations are done by RPython +and you're only left with the (hopefully few) hard and interesting bits. + +The core of STM works as a library written in C (see `reference to +implementation details`_ below). It means that it can be used on +other interpreters than the ones produced by RPython. Duhton_ is an +early example of that. At this point, you might think about adapting +this library for CPython. You're warned, though: as far as I can +tell, it is a doomed idea. I had a hard time debugging Duhton, and +that's infinitely simpler than CPython. Even ignoring that, you can +see in the C sources of Duhton that many core design decisions are +different than in CPython: no refcounting; limited support for +prebuilt "static" objects; ``stm_read()`` and ``stm_write()`` macro +calls everywhere (and getting very rare and very obscure bugs if you +forget one); and so on. You could imagine some custom special-purpose +extension of the C language, which you would preprocess to regular C. +In my opinion that's starting to look a lot like RPython itself, but +maybe you'd prefer this approach. Of course you still have to worry +about each and every C extension module you need, but maybe you'd have +a way forward. + +.. _Duhton: https://bitbucket.org/pypy/duhton User Guide ========== +How to write multithreaded programs: the 10'000-feet view +--------------------------------------------------------- + +PyPy-STM offers two ways to write multithreaded programs: + +* the traditional way, using the ``thread`` or ``threading`` modules, + described first__. + +* using ``TransactionQueue``, described next__, as a way to hide the + low-level notion of threads. + +.. __: `Drop-in replacement`_ +.. __: `transaction.TransactionQueue`_ + +The issues with low-level threads are well known (particularly in other +languages that don't have GIL-based interpreters): memory corruption, +deadlocks, livelocks, and so on. There are alternative approaches to +dealing directly with threads, like OpenMP_. These approaches +typically enforce some structure on your code. ``TransactionQueue`` +is in part similar: your program needs to have "some chances" of +parallelization before you can apply it. But I believe that the scope +of applicability is much larger with ``TransactionQueue`` than with +other approaches. It usually works without forcing a complete +reorganization of your existing code, and it works on any Python +program which has got *latent* and *imperfect* parallelism. Ideally, +it only requires that the end programmer identifies where this +parallelism is likely to be found, and communicates it to the system +using a simple API. + +.. _OpenMP: http://en.wikipedia.org/wiki/OpenMP + + Drop-in replacement ------------------- @@ -175,29 +271,199 @@ This works by internally considering the points where a standard PyPy or CPython would release the GIL, and replacing them with the boundaries of -"transaction". Like their database equivalent, multiple transactions +"transactions". Like their database equivalent, multiple transactions can execute in parallel, but will commit in some serial order. They appear to behave as if they were completely run in this serialization order. +transaction.TransactionQueue +---------------------------- + +In CPU-hungry programs, we can often easily identify outermost loops +over some data structure, or other repetitive algorithm, where each +"block" consists of processing a non-trivial amount of data, and where +the blocks "have a good chance" to be independent from each other. We +don't need to prove that they are actually independent: it is enough +if they are *often independent* --- or, more precisely, if we *think +they should be* often independent. + +One typical example would look like this, where the function ``func()`` +typically invokes a large amount of code:: + + for key, value in bigdict.items(): + func(key, value) + +Then you simply replace the loop with:: + + from transaction import TransactionQueue + + tr = TransactionQueue() + for key, value in bigdict.items(): + tr.add(func, key, value) + tr.run() + +This code's behavior is equivalent. Internally, the +``TransactionQueue`` object will start N threads and try to run the +``func(key, value)`` calls on all threads in parallel. But note the +difference with a regular thread-pooling library, as found in many +lower-level languages than Python: the function calls are not randomly +interleaved with each other just because they run in parallel. The +behavior did not change because we are using ``TransactionQueue``. +All the calls still *appear* to execute in some serial order. + +A typical usage of ``TransactionQueue`` goes like that: at first, +the performance does not increase. +In fact, it is likely to be worse. Typically, this is +indicated by the total CPU usage, which remains low (closer to 1 than +N cores). First note that it is expected that the CPU usage should +not go much higher than 1 in the JIT warm-up phase: you must run a +program for several seconds, or for larger programs at least one +minute, to give the JIT a chance to warm up enough. But if CPU usage +remains low even afterwards, then the ``PYPYSTM`` environment variable +can be used to track what is going on. + +Run your program with ``PYPYSTM=logfile`` to produce a log file called +``logfile``. Afterwards, use the ``pypy/stm/print_stm_log.py`` +utility to inspect the content of this log file. It produces output +like this (sorted by amount of time lost, largest first):: + + 10.5s lost in aborts, 1.25s paused (12412x STM_CONTENTION_WRITE_WRITE) + File "foo.py", line 10, in f + someobj.stuff = 5 + File "bar.py", line 20, in g + someobj.other = 10 + +This means that 10.5 seconds were lost running transactions that were +aborted (which caused another 1.25 seconds of lost time by pausing), +because of the reason shown in the two independent single-entry +tracebacks: one thread ran the line ``someobj.stuff = 5``, whereas +another thread concurrently ran the line ``someobj.other = 10`` on the +same object. These two writes are done to the same object. This +causes a conflict, which aborts one of the two transactions. In the +example above this occurred 12412 times. + +The two other conflict sources are ``STM_CONTENTION_INEVITABLE``, +which means that two transactions both tried to do an external +operation, like printing or reading from a socket or accessing an +external array of raw data; and ``STM_CONTENTION_WRITE_READ``, which +means that one transaction wrote to an object but the other one merely +read it, not wrote to it (in that case only the writing transaction is +reported; the location for the reads is not recorded because doing so +is not possible without a very large performance impact). + +Common causes of conflicts: + +* First of all, any I/O or raw manipulation of memory turns the + transaction inevitable ("must not abort"). There can be only one + inevitable transaction running at any time. A common case is if + each transaction starts with sending data to a log file. You should + refactor this case so that it occurs either near the end of the + transaction (which can then mostly run in non-inevitable mode), or + delegate it to a separate transaction or even a separate thread. + +* Writing to a list or a dictionary conflicts with any read from the + same list or dictionary, even one done with a different key. For + dictionaries and sets, you can try the types ``transaction.stmdict`` + and ``transaction.stmset``, which behave mostly like ``dict`` and + ``set`` but allow concurrent access to different keys. (What is + missing from them so far is lazy iteration: for example, + ``stmdict.iterkeys()`` is implemented as ``iter(stmdict.keys())``; + and, unlike PyPy's dictionaries and sets, the STM versions are not + ordered.) There are also experimental ``stmiddict`` and + ``stmidset`` classes using the identity of the key. + +* ``time.time()`` and ``time.clock()`` turn the transaction inevitable + in order to guarantee that a call that appears to be later will really + return a higher number. If getting slightly unordered results is + fine, use ``transaction.time()`` or ``transaction.clock()``. The + latter operations guarantee to return increasing results only if you + can "prove" that two calls occurred in a specific order (for example + because they are both called by the same thread). In cases where no + such proof is possible, you might get randomly interleaved values. + (If you have two independent transactions, they normally behave as if + one of them was fully executed before the other; but using + ``transaction.time()`` you might see the "hidden truth" that they are + actually interleaved.) + +* ``transaction.threadlocalproperty`` can be used at class-level:: + + class Foo(object): # must be a new-style class! + x = transaction.threadlocalproperty() + y = transaction.threadlocalproperty(dict) + + This declares that instances of ``Foo`` have two attributes ``x`` + and ``y`` that are thread-local: reading or writing them from + concurrently-running transactions will return independent results. + (Any other attributes of ``Foo`` instances will be globally visible + from all threads, as usual.) This is useful together with + ``TransactionQueue`` for these two cases: + + - For attributes of long-lived objects that change during one + transaction, but should always be reset to some initial value + around transaction (for example, initialized to 0 at the start of + a transaction; or, if used for a list of pending things to do + within this transaction, it will always be empty at the end of one + transaction). + + - For general caches across transactions. With ``TransactionQueue`` + you get a pool of a fixed number N of threads, each running the + transactions serially. A thread-local property will have the + value last stored in it by the same thread, which may come from a + random previous transaction. Basically, you get N copies of the + property's value, and each transaction accesses a random copy. It + works fine for caches. + + In more details, the optional argument to ``threadlocalproperty()`` + is the default value factory: in case no value was assigned in the + current thread yet, the factory is called and its result becomes the + value in that thread (like ``collections.defaultdict``). If no + default value factory is specified, uninitialized reads raise + ``AttributeError``. + +* In addition to all of the above, there are cases where write-write + conflicts are caused by writing the same value to an attribute again + and again. See for example ea2e519614ab_: this fixes two such + issues where we write an object field without first checking if we + already did it. The ``dont_change_any_more`` field is a flag set to + ``True`` in that part of the code, but usually this + ``rtyper_makekey()`` method will be called many times for the same + object; the code used to repeatedly set the flag to ``True``, but + now it first checks and only does the write if it is ``False``. + Similarly, in the second half of the checkin, the method + ``setup_block_entry()`` used to both assign the ``concretetype`` + fields and return a list, but its two callers were different: one + would really need the ``concretetype`` fields initialized, whereas + the other would only need to get its result list --- the + ``concretetype`` field in that case might already be set or not, but + that would not matter. + +.. _ea2e519614ab: https://bitbucket.org/pypy/pypy/commits/ea2e519614ab + +Note that Python is a complicated language; there are a number of less +common cases that may cause conflict (of any kind) where we might not +expect it at priori. In many of these cases it could be fixed; please +report any case that you don't understand. + + Atomic sections --------------- -PyPy supports *atomic sections,* which are blocks of code which you want -to execute without "releasing the GIL". *This is experimental and may -be removed in the future.* In STM terms, this means blocks of code that -are executed while guaranteeing that the transaction is not interrupted -in the middle. +The ``TransactionQueue`` class described above is based on *atomic +sections,* which are blocks of code which you want to execute without +"releasing the GIL". In STM terms, this means blocks of code that are +executed while guaranteeing that the transaction is not interrupted in +the middle. *This is experimental and may be removed in the future* +if `Software lock elision`_ is ever implemented. -Here is a usage example:: +Here is a direct usage example:: - with __pypy__.thread.atomic: + with transaction.atomic: assert len(lst1) == 10 x = lst1.pop(0) lst1.append(x) -In this (bad) example, we are sure that the item popped off one end of +In this example, we are sure that the item popped off one end of the list is appened again at the other end atomically. It means that another thread can run ``len(lst1)`` or ``x in lst1`` without any particular synchronization, and always see the same results, @@ -221,25 +487,27 @@ it likely that such a piece of code will eventually block all other threads anyway. -Note that if you want to experiment with ``atomic``, you may have to add -manually a transaction break just before the atomic block. This is +Note that if you want to experiment with ``atomic``, you may have to +manually add a transaction break just before the atomic block. This is because the boundaries of the block are not guaranteed to be the boundaries of the transaction: the latter is at least as big as the -block, but maybe bigger. Therefore, if you run a big atomic block, it +block, but may be bigger. Therefore, if you run a big atomic block, it is a good idea to break the transaction just before. This can be done -e.g. by the hack of calling ``time.sleep(0)``. (This may be fixed at +by calling ``transaction.hint_commit_soon()``. (This may be fixed at some point.) -There are also issues with the interaction of locks and atomic blocks. -This can be seen if you write to files (which have locks), including -with a ``print`` to standard output. If one thread tries to acquire a -lock while running in an atomic block, and another thread has got the -same lock, then the former may fail with a ``thread.error``. The reason -is that "waiting" for some condition to become true --while running in -an atomic block-- does not really make sense. For now you can work -around it by making sure that, say, all your prints are either in an -``atomic`` block or none of them are. (This kind of issue is -theoretically hard to solve.) +There are also issues with the interaction of regular locks and atomic +blocks. This can be seen if you write to files (which have locks), +including with a ``print`` to standard output. If one thread tries to +acquire a lock while running in an atomic block, and another thread +has got the same lock at that point, then the former may fail with a +``thread.error``. (Don't rely on it; it may also deadlock.) +The reason is that "waiting" for some condition to +become true --while running in an atomic block-- does not really make +sense. For now you can work around it by making sure that, say, all +your prints are either in an ``atomic`` block or none of them are. +(This kind of issue is theoretically hard to solve and may be the +reason for atomic block support to eventually be removed.) Locks @@ -293,106 +561,47 @@ .. _`software lock elision`: https://www.repository.cam.ac.uk/handle/1810/239410 -Atomic sections, Transactions, etc.: a better way to write parallel programs ----------------------------------------------------------------------------- +Miscellaneous functions +----------------------- -(This section is based on locks as we plan to implement them, but also -works with the existing atomic sections.) +* First, note that the ``transaction`` module is found in the file + ``lib_pypy/transaction.py``. This file can be copied around to + execute the same programs on CPython or on non-STM PyPy, with + fall-back behavior. (One case where the behavior differs is + ``atomic``, which is in this fall-back case just a regular lock; so + ``with atomic`` only prevent other threads from entering other + ``with atomic`` sections, but won't prevent other threads from + running non-atomic code.) -In the cases where elision works, the block of code can run in parallel -with other blocks of code *even if they are protected by the same lock.* -You still get the illusion that the blocks are run sequentially. This -works even for multiple threads that run each a series of such blocks -and nothing else, protected by one single global lock. This is -basically the Python application-level equivalent of what was done with -the interpreter in ``pypy-stm``: while you think you are writing -thread-unfriendly code because of this global lock, actually the -underlying system is able to make it run on multiple cores anyway. - -This capability can be hidden in a library or in the framework you use; -the end user's code does not need to be explicitly aware of using -threads. For a simple example of this, there is `transaction.py`_ in -``lib_pypy``. The idea is that you write, or already have, some program -where the function ``f(key, value)`` runs on every item of some big -dictionary, say:: - - for key, value in bigdict.items(): - f(key, value) - -Then you simply replace the loop with:: - - for key, value in bigdict.items(): - transaction.add(f, key, value) - transaction.run() - -This code runs the various calls to ``f(key, value)`` using a thread -pool, but every single call is executed under the protection of a unique -lock. The end result is that the behavior is exactly equivalent --- in -fact it makes little sense to do it in this way on a non-STM PyPy or on -CPython. But on ``pypy-stm``, the various locked calls to ``f(key, -value)`` can tentatively be executed in parallel, even if the observable -result is as if they were executed in some serial order. - -This approach hides the notion of threads from the end programmer, -including all the hard multithreading-related issues. This is not the -first alternative approach to explicit threads; for example, OpenMP_ is -one. However, it is one of the first ones which does not require the -code to be organized in a particular fashion. Instead, it works on any -Python program which has got latent, imperfect parallelism. Ideally, it -only requires that the end programmer identifies where this parallelism -is likely to be found, and communicates it to the system, using for -example the ``transaction.add()`` scheme. - -.. _`transaction.py`: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/lib_pypy/transaction.py -.. _OpenMP: http://en.wikipedia.org/wiki/OpenMP - - -.. _`transactional_memory`: - -API of transactional_memory ---------------------------- - -The new pure Python module ``transactional_memory`` runs on both CPython -and PyPy, both with and without STM. It contains: - -* ``getsegmentlimit()``: return the number of "segments" in +* ``transaction.getsegmentlimit()``: return the number of "segments" in this pypy-stm. This is the limit above which more threads will not be able to execute on more cores. (Right now it is limited to 4 due to inter-segment overhead, but should be increased in the future. It should also be settable, and the default value should depend on the number of actual CPUs.) If STM is not available, this returns 1. -* ``print_abort_info(minimum_time=0.0)``: debugging help. Each thread - remembers the longest abort or pause it did because of cross-thread - contention_. This function prints it to ``stderr`` if the time lost - is greater than ``minimum_time`` seconds. The record is then - cleared, to make it ready for new events. This function returns - ``True`` if it printed a report, and ``False`` otherwise. +* ``__pypy__.thread.signals_enabled``: a context manager that runs its + block of code with signals enabled. By default, signals are only + enabled in the main thread; a non-main thread will not receive + signals (this is like CPython). Enabling signals in non-main + threads is useful for libraries where threads are hidden and the end + user is not expecting his code to run elsewhere than in the main + thread. +* ``pypystm.exclusive_atomic``: a context manager similar to + ``transaction.atomic`` but which complains if it is nested. -API of __pypy__.thread ----------------------- +* ``transaction.is_atomic()``: return True if called from an atomic + context. -The ``__pypy__.thread`` submodule is a built-in module of PyPy that -contains a few internal built-in functions used by the -``transactional_memory`` module, plus the following: +* ``pypystm.count()``: return a different positive integer every time + it is called. This works without generating conflicts. The + returned integers are only roughly in increasing order; this should + not be relied upon. -* ``__pypy__.thread.atomic``: a context manager to run a block in - fully atomic mode, without "releasing the GIL". (May be eventually - removed?) -* ``__pypy__.thread.signals_enabled``: a context manager that runs its - block with signals enabled. By default, signals are only enabled in - the main thread; a non-main thread will not receive signals (this is - like CPython). Enabling signals in non-main threads is useful for - libraries where threads are hidden and the end user is not expecting - his code to run elsewhere than in the main thread. - - -.. _contention: - -Conflicts ---------- +More details about conflicts +---------------------------- Based on Software Transactional Memory, the ``pypy-stm`` solution is prone to "conflicts". To repeat the basic idea, threads execute their code @@ -408,25 +617,26 @@ the transaction). If this occurs too often, parallelization fails. How much actual parallelization a multithreaded program can see is a bit -subtle. Basically, a program not using ``__pypy__.thread.atomic`` or +subtle. Basically, a program not using ``transaction.atomic`` or eliding locks, or doing so for very short amounts of time, will parallelize almost freely (as long as it's not some artificial example where, say, all threads try to increase the same global counter and do nothing else). -However, using if the program requires longer transactions, it comes +However, if the program requires longer transactions, it comes with less obvious rules. The exact details may vary from version to version, too, until they are a bit more stabilized. Here is an overview. Parallelization works as long as two principles are respected. The -first one is that the transactions must not *conflict* with each other. -The most obvious sources of conflicts are threads that all increment a -global shared counter, or that all store the result of their -computations into the same list --- or, more subtly, that all ``pop()`` -the work to do from the same list, because that is also a mutation of -the list. (It is expected that some STM-aware library will eventually -be designed to help with conflict problems, like a STM-aware queue.) +first one is that the transactions must not *conflict* with each +other. The most obvious sources of conflicts are threads that all +increment a global shared counter, or that all store the result of +their computations into the same list --- or, more subtly, that all +``pop()`` the work to do from the same list, because that is also a +mutation of the list. (You can work around it with +``transaction.stmdict``, but for that specific example, some STM-aware +queue should eventually be designed.) A conflict occurs as follows: when a transaction commits (i.e. finishes successfully) it may cause other transactions that are still in progress @@ -442,22 +652,23 @@ Another issue is that of avoiding long-running so-called "inevitable" transactions ("inevitable" is taken in the sense of "which cannot be avoided", i.e. transactions which cannot abort any more). Transactions -like that should only occur if you use ``__pypy__.thread.atomic``, -generally become of I/O in atomic blocks. They work, but the +like that should only occur if you use ``atomic``, +generally because of I/O in atomic blocks. They work, but the transaction is turned inevitable before the I/O is performed. For all the remaining execution time of the atomic block, they will impede parallel work. The best is to organize the code so that such operations -are done completely outside ``__pypy__.thread.atomic``. +are done completely outside ``atomic``. -(This is related to the fact that blocking I/O operations are +(This is not unrelated to the fact that blocking I/O operations are discouraged with Twisted, and if you really need them, you should do them on their own separate thread.) -In case of lock elision, we don't get long-running inevitable -transactions, but a different problem can occur: doing I/O cancels lock -elision, and the lock turns into a real lock, preventing other threads -from committing if they also need this lock. (More about it when lock -elision is implemented and tested.) +In case lock elision eventually replaces atomic sections, we wouldn't +get long-running inevitable transactions, but the same problem occurs +in a different way: doing I/O cancels lock elision, and the lock turns +into a real lock. This prevents other threads from committing if they +also need this lock. (More about it when lock elision is implemented +and tested.) @@ -467,56 +678,30 @@ XXX this section mostly empty for now -Low-level statistics --------------------- +Technical reports +----------------- -When a non-main thread finishes, you get low-level statistics printed to -stderr, looking like that:: +STMGC-C7 is described in detail in a `technical report`__. - thread 0x7f73377fe600: - outside transaction 42182 0.506 s - run current 85466 0.000 s - run committed 34262 3.178 s - run aborted write write 6982 0.083 s - run aborted write read 550 0.005 s - run aborted inevitable 388 0.010 s - run aborted other 0 0.000 s - wait free segment 0 0.000 s - wait write read 78 0.027 s - wait inevitable 887 0.490 s - wait other 0 0.000 s - sync commit soon 1 0.000 s - bookkeeping 51418 0.606 s - minor gc 162970 1.135 s - major gc 1 0.019 s - sync pause 59173 1.738 s - longest recordered marker 0.000826 s - "File "x.py", line 5, in f" +A separate `position paper`__ gives an overview of our position about +STM in general. -On each line, the first number is a counter, and the second number gives -the associated time --- the amount of real time that the thread was in -this state. The sum of all the times should be equal to the total time -between the thread's start and the thread's end. The most important -points are "run committed", which gives the amount of useful work, and -"outside transaction", which should give the time spent e.g. in library -calls (right now it seems to be larger than that; to investigate). The -various "run aborted" and "wait" entries are time lost due to -conflicts_. Everything else is overhead of various forms. (Short-, -medium- and long-term future work involves reducing this overhead :-) - -The last two lines are special; they are an internal marker read by -``transactional_memory.print_abort_info()``. +.. __: http://bitbucket.org/pypy/extradoc/src/extradoc/talk/dls2014/paper/paper.pdf +.. __: http://bitbucket.org/pypy/extradoc/src/extradoc/talk/icooolps2014/ Reference to implementation details ----------------------------------- -The core of the implementation is in a separate C library called stmgc_, -in the c7_ subdirectory. Please see the `README.txt`_ for more -information. In particular, the notion of segment is discussed there. +The core of the implementation is in a separate C library called +stmgc_, in the c7_ subdirectory (current version of pypy-stm) and in +the c8_ subdirectory (bleeding edge version). Please see the +`README.txt`_ for more information. In particular, the notion of +segment is discussed there. .. _stmgc: https://bitbucket.org/pypy/stmgc/src/default/ .. _c7: https://bitbucket.org/pypy/stmgc/src/default/c7/ +.. _c8: https://bitbucket.org/pypy/stmgc/src/default/c8/ .. _`README.txt`: https://bitbucket.org/pypy/stmgc/raw/default/c7/README.txt PyPy itself adds on top of it the automatic placement of read__ and write__ diff --git a/pypy/doc/whatsnew-2.5.0.rst b/pypy/doc/whatsnew-2.5.0.rst --- a/pypy/doc/whatsnew-2.5.0.rst +++ b/pypy/doc/whatsnew-2.5.0.rst @@ -1,6 +1,6 @@ -======================= -What's new in PyPy 2.5 -======================= +======================== +What's new in PyPy 2.5.0 +======================== .. this is a revision shortly after release-2.4.x .. startrev: 7026746cbb1b diff --git a/pypy/doc/whatsnew-2.5.1.rst b/pypy/doc/whatsnew-2.5.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-2.5.1.rst @@ -0,0 +1,47 @@ +======================== +What's new in PyPy 2.5.1 +======================== + +.. this is a revision shortly after release-2.5.0 +.. startrev: 397b96217b85 + + +Non-blocking file reads sometimes raised EAGAIN even though they +had buffered data waiting, fixed in b1c4fcb04a42 + +Fix a bug in cpyext in multithreded programs acquiring/releasing the GIL + +.. branch: vmprof + +.. branch: stackroot-speedup-2 + +Avoid tracing all stack roots during repeated minor collections, +by ignoring the part of the stack that didn't change + +.. branch: stdlib-2.7.9 + +Update stdlib to version 2.7.9 + +.. branch: fix-kqueue-error2 + +Fix exception being raised by kqueue.control (CPython compatibility) + +.. branch: gitignore + +.. branch: framestate2 + +Refactor rpython.flowspace.framestate.FrameState. + +.. branch: alt_errno + +Add an alternative location to save LastError, errno around ctypes, +cffi external calls so things like pdb will not overwrite it + +.. branch: nonquadratic-heapcache + +Speed up the warmup times of the JIT by removing a quadratic algorithm in the +heapcache. + +.. branch: online-transforms-2 + +Simplify flow graphs on the fly during annotation phase. diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -2,24 +2,22 @@ What's new in PyPy 2.5+ ======================= -.. this is a revision shortly after release-2.5.x -.. startrev: 397b96217b85 +.. this is a revision shortly after release-2.5.1 +.. startrev: cb01edcb59414d9d93056e54ed060673d24e67c1 +Issue #2017: on non-Linux-x86 platforms, reduced the memory impact of +creating a lot of greenlets/tasklets. Particularly useful on Win32 and +on ARM, where you used to get a MemoryError after only 2500-5000 +greenlets (the 32-bit address space is exhausted). -Non-blocking file reads sometimes raised EAGAIN even though they -had buffered data waiting, fixed in b1c4fcb04a42 +.. branch: gc-incminimark-pinning-improve +Object Pinning is now used in `bz2` and `rzlib` (therefore also affects +Python's `zlib`). In case the data to compress/decompress is inside the nursery +(incminimark) it no longer needs to create a non-moving copy of it. This saves +one `malloc` and copying the data. Additionally a new GC environment variable +is introduced (`PYPY_GC_MAX_PINNED`) primarily for debugging purposes. - -.. branch: vmprof - -.. branch: stackroot-speedup-2 -Avoid tracing all stack roots during repeated minor collections, -by ignoring the part of the stack that didn't change - -.. branch: stdlib-2.7.9 -Update stdlib to version 2.7.9 - -.. branch: fix-kqueue-error2 -Fix exception being raised by kqueue.control (CPython compatibility) - -.. branch: gitignore +.. branch: refactor-pycall +Make `*`-unpacking in RPython function calls completely equivalent to passing +the tuple's elements as arguments. In other words, `f(*(a, b))` now behaves +exactly like `f(a, b)`. diff --git a/pypy/goal/getnightly.py b/pypy/goal/getnightly.py --- a/pypy/goal/getnightly.py +++ b/pypy/goal/getnightly.py @@ -7,7 +7,7 @@ if sys.platform.startswith('linux'): arch = 'linux' cmd = 'wget "%s"' - tar = "tar -x -v --wildcards --strip-components=2 -f %s '*/bin/pypy'" + tar = "tar -x -v --wildcards --strip-components=2 -f %s '*/bin/pypy' '*/bin/libpypy-c.so'" if os.uname()[-1].startswith('arm'): arch += '-armhf-raspbian' elif sys.platform.startswith('darwin'): diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -106,6 +106,9 @@ space.call_function(w_pathsetter, w_path) # import site try: + space.setattr(space.getbuiltinmodule('sys'), + space.wrap('executable'), + space.wrap(home)) import_ = space.getattr(space.getbuiltinmodule('__builtin__'), space.wrap('__import__')) space.call_function(import_, space.wrap('site')) @@ -138,7 +141,7 @@ res = _pypy_execute_source(source) before = rffi.aroundstate.before if before: before() - return rffi.cast(rffi.INT, res) + return rffi.cast(rffi.INT, res) @entrypoint('main', [], c_name='pypy_init_threads') def pypy_init_threads(): @@ -309,7 +312,7 @@ w_dict = app.getwdict(space) entry_point, _ = create_entry_point(space, w_dict) - return entry_point, None, PyPyAnnotatorPolicy(single_space = space) + return entry_point, None, PyPyAnnotatorPolicy() def interface(self, ns): for name in ['take_options', 'handle_config', 'print_help', 'target', diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -1,5 +1,5 @@ #! /usr/bin/env python -# App-level version of py.py. +# This is pure Python code that handles the main entry point into "pypy". # See test/test_app_main. # Missing vs CPython: -d, -t, -v, -x, -3 @@ -157,10 +157,13 @@ current = group raise SystemExit +def get_sys_executable(): + return getattr(sys, 'executable', 'pypy') + def print_help(*args): import os print 'usage: %s [option] ... [-c cmd | -m mod | file | -] [arg] ...' % ( - sys.executable,) + get_sys_executable(),) print USAGE1, if 'pypyjit' in sys.builtin_module_names: print "--jit options: advanced JIT options: try 'off' or 'help'" @@ -171,7 +174,7 @@ try: import pypyjit except ImportError: - print >> sys.stderr, "No jit support in %s" % (sys.executable,) + print >> sys.stderr, "No jit support in %s" % (get_sys_executable(),) return items = sorted(pypyjit.defaults.items()) print 'Advanced JIT options: a comma-separated list of OPTION=VALUE:' @@ -209,7 +212,7 @@ raise SystemExit if 'pypyjit' not in sys.builtin_module_names: print >> sys.stderr, ("Warning: No jit support in %s" % - (sys.executable,)) + (get_sys_executable(),)) else: import pypyjit pypyjit.set_param(jitparam) @@ -219,8 +222,8 @@ def print_error(msg): print >> sys.stderr, msg - print >> sys.stderr, 'usage: %s [options]' % (sys.executable,) - print >> sys.stderr, 'Try `%s -h` for more information.' % (sys.executable,) + print >> sys.stderr, 'usage: %s [options]' % (get_sys_executable(),) + print >> sys.stderr, 'Try `%s -h` for more information.' % (get_sys_executable(),) def fdopen(fd, mode, bufsize=-1): try: @@ -514,6 +517,10 @@ elif not sys.stdout.isatty(): set_fully_buffered_io() + if we_are_translated(): + import __pypy__ + __pypy__.save_module_content_for_future_reload(sys) + mainmodule = type(sys)('__main__') sys.modules['__main__'] = mainmodule diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py --- a/pypy/interpreter/astcompiler/assemble.py +++ b/pypy/interpreter/astcompiler/assemble.py @@ -648,7 +648,7 @@ def _compute_UNPACK_SEQUENCE(arg): - return arg + 1 + return arg - 1 def _compute_DUP_TOPX(arg): return arg diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py --- a/pypy/interpreter/astcompiler/test/test_compiler.py +++ b/pypy/interpreter/astcompiler/test/test_compiler.py @@ -759,6 +759,19 @@ """ self.simple_test(source, 'l', [1, 2]) + def test_unpack_wrong_stackeffect(self): + source = """if 1: + l = [1, 2] + a, b = l + a, b = l + a, b = l + a, b = l + a, b = l + a, b = l + """ + code = compile_with_astcompiler(source, 'exec', self.space) + assert code.co_stacksize == 2 + def test_lambda(self): yield self.st, "y = lambda x: x", "y(4)", 4 diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py --- a/pypy/interpreter/mixedmodule.py +++ b/pypy/interpreter/mixedmodule.py @@ -55,7 +55,10 @@ if self.w_initialdict is None: Module.init(self, space) if not self.lazy and self.w_initialdict is None: - self.w_initialdict = space.call_method(self.w_dict, 'items') + self.save_module_content_for_future_reload() + + def save_module_content_for_future_reload(self): + self.w_initialdict = self.space.call_method(self.w_dict, 'items') def get_applevel_name(cls): @@ -119,7 +122,7 @@ w_value = self.get(name) space.setitem(self.w_dict, space.new_interned_str(name), w_value) self.lazy = False - self.w_initialdict = space.call_method(self.w_dict, 'items') + self.save_module_content_for_future_reload() return self.w_dict def _cleanup_(self): diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -4,7 +4,7 @@ The bytecode interpreter itself is implemented by the PyFrame class. """ -import dis, imp, struct, types, new, sys +import dis, imp, struct, types, new, sys, os from pypy.interpreter import eval from pypy.interpreter.signature import Signature @@ -128,6 +128,17 @@ if (self.magic == cpython_magic and '__pypy__' not in sys.builtin_module_names): raise Exception("CPython host codes should not be rendered") + # When translating PyPy, freeze the file name + # /lastdirname/basename.py + # instead of freezing the complete translation-time path. + filename = self.co_filename.lstrip('<').rstrip('>') + if filename.lower().endswith('.pyc'): + filename = filename[:-1] + basename = os.path.basename(filename) + lastdirname = os.path.basename(os.path.dirname(filename)) + if lastdirname: + basename = '%s/%s' % (lastdirname, basename) + self.co_filename = '/%s' % (basename,) co_names = property(lambda self: [self.space.unwrap(w_name) for w_name in self.co_names_w]) # for trace diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -1619,6 +1619,13 @@ def prepare_exec(f, prog, globals, locals, compile_flags, builtin, codetype): """Manipulate parameters to exec statement to (codeobject, dict, dict). """ + if (globals is None and locals is None and + isinstance(prog, tuple) and + (len(prog) == 2 or len(prog) == 3)): + globals = prog[1] + if len(prog) == 3: + locals = prog[2] + prog = prog[0] if globals is None: globals = f.f_globals if locals is None: diff --git a/pypy/interpreter/test/test_exec.py b/pypy/interpreter/test/test_exec.py --- a/pypy/interpreter/test/test_exec.py +++ b/pypy/interpreter/test/test_exec.py @@ -262,3 +262,11 @@ """] for c in code: compile(c, "

", "exec")
+
+    def test_exec_tuple(self):
+        # note: this is VERY different than testing exec("a = 42", d), because
+        # this specific case is handled specially by the AST compiler
+        d = {}
+        x = ("a = 42", d)
+        exec x
+        assert d['a'] == 42
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -373,7 +373,7 @@
         config = make_config(None)
         space = make_objspace(config)
         w_executable = space.wrap('executable')
-        assert space.str_w(space.getattr(space.sys, w_executable)) == 'py.py'
+        assert space.findattr(space.sys, w_executable) is None
         space.setattr(space.sys, w_executable, space.wrap('foobar'))
         assert space.str_w(space.getattr(space.sys, w_executable)) == 'foobar'
         space.startup()
diff --git a/pypy/interpreter/test/test_targetpypy.py b/pypy/interpreter/test/test_targetpypy.py
--- a/pypy/interpreter/test/test_targetpypy.py
+++ b/pypy/interpreter/test/test_targetpypy.py
@@ -8,7 +8,7 @@
         entry_point = get_entry_point(config)[0]
         entry_point(['pypy-c' , '-S', '-c', 'print 3'])
 
-def test_exeucte_source(space):
+def test_execute_source(space):
     _, d = create_entry_point(space, None)
     execute_source = d['pypy_execute_source']
     lls = rffi.str2charp("import sys; sys.modules['xyz'] = 3")
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -24,13 +24,9 @@
         self.end = end
         self.reason = reason
 
- at specialize.memo()
-def rpy_encode_error_handler():
-    # A RPython version of the "strict" error handler.
-    def raise_unicode_exception_encode(errors, encoding, msg, u,
-                                       startingpos, endingpos):
-        raise RUnicodeEncodeError(encoding, u, startingpos, endingpos, msg)
-    return raise_unicode_exception_encode
+def raise_unicode_exception_encode(errors, encoding, msg, u,
+                                   startingpos, endingpos):
+    raise RUnicodeEncodeError(encoding, u, startingpos, endingpos, msg)
 
 # ____________________________________________________________
 
@@ -67,5 +63,5 @@
     # This is not the case with Python3.
     return runicode.unicode_encode_utf_8(
         uni, len(uni), "strict",
-        errorhandler=rpy_encode_error_handler(),
+        errorhandler=raise_unicode_exception_encode,
         allow_surrogates=True)
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -82,6 +82,8 @@
         'strategy'                  : 'interp_magic.strategy',  # dict,set,list
         'set_debug'                 : 'interp_magic.set_debug',
         'locals_to_fast'            : 'interp_magic.locals_to_fast',
+        'save_module_content_for_future_reload':
+                          'interp_magic.save_module_content_for_future_reload',
     }
     if sys.platform == 'win32':
         interpleveldefs['get_console_cp'] = 'interp_magic.get_console_cp'
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -1,6 +1,7 @@
 from pypy.interpreter.error import OperationError, wrap_oserror
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.pyframe import PyFrame
+from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rlib.objectmodel import we_are_translated
 from pypy.objspace.std.dictmultiobject import W_DictMultiObject
 from pypy.objspace.std.listobject import W_ListObject
@@ -130,3 +131,7 @@
 def locals_to_fast(space, w_frame):
     assert isinstance(w_frame, PyFrame)
     w_frame.locals2fast()
+
+ at unwrap_spec(w_module=MixedModule)
+def save_module_content_for_future_reload(space, w_module):
+    w_module.save_module_content_for_future_reload()
diff --git a/pypy/module/__pypy__/test/test_magic.py b/pypy/module/__pypy__/test/test_magic.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__pypy__/test/test_magic.py
@@ -0,0 +1,15 @@
+
+class AppTestMagic:
+    spaceconfig = dict(usemodules=['__pypy__'])
+
+    def test_save_module_content_for_future_reload(self):
+        import sys, __pypy__
+        d = sys.dont_write_bytecode
+        sys.dont_write_bytecode = "hello world"
+        __pypy__.save_module_content_for_future_reload(sys)
+        sys.dont_write_bytecode = d
+        reload(sys)
+        assert sys.dont_write_bytecode == "hello world"
+        #
+        sys.dont_write_bytecode = d
+        __pypy__.save_module_content_for_future_reload(sys)
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -2,13 +2,15 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rlib import rdynload
 
+VERSION = "0.9.2"
+
 
 class Module(MixedModule):
 
     appleveldefs = {
         }
     interpleveldefs = {
-        '__version__': 'space.wrap("0.8.6+")',
+        '__version__': 'space.wrap("%s")' % VERSION,
 
         'load_library': 'libraryobj.load_library',
 
diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -81,4 +81,5 @@
     if size < 0:
         raise oefmt(space.w_TypeError,
                     "don't know the size pointed to by '%s'", ctype.name)
-    return space.wrap(MiniBuffer(LLBuffer(w_cdata._cdata, size), w_cdata))
+    ptr = w_cdata.unsafe_escaping_ptr()    # w_cdata kept alive by MiniBuffer()
+    return space.wrap(MiniBuffer(LLBuffer(ptr, size), w_cdata))
diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py
--- a/pypy/module/_cffi_backend/ccallback.py
+++ b/pypy/module/_cffi_backend/ccallback.py
@@ -48,9 +48,12 @@
             raise oefmt(space.w_NotImplementedError,
                         "%s: callback with unsupported argument or "
                         "return type or with '...'", self.getfunctype().name)
-        res = clibffi.c_ffi_prep_closure(self.get_closure(), cif_descr.cif,
-                                         invoke_callback,
-                                         rffi.cast(rffi.VOIDP, self.unique_id))
+        with self as ptr:
+            closure_ptr = rffi.cast(clibffi.FFI_CLOSUREP, ptr)
+            unique_id = rffi.cast(rffi.VOIDP, self.unique_id)
+            res = clibffi.c_ffi_prep_closure(closure_ptr, cif_descr.cif,
+                                             invoke_callback,
+                                             unique_id)
         if rffi.cast(lltype.Signed, res) != clibffi.FFI_OK:
             raise OperationError(space.w_SystemError,
                 space.wrap("libffi failed to build this callback"))
@@ -62,12 +65,9 @@
             from pypy.module.thread.os_thread import setup_threads
             setup_threads(space)
 
-    def get_closure(self):
-        return rffi.cast(clibffi.FFI_CLOSUREP, self._cdata)
-
     #@rgc.must_be_light_finalizer
     def __del__(self):
-        clibffi.closureHeap.free(self.get_closure())
+        clibffi.closureHeap.free(rffi.cast(clibffi.FFI_CLOSUREP, self._ptr))

From noreply at buildbot.pypy.org  Thu Apr  9 14:02:28 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  9 Apr 2015 14:02:28 +0200 (CEST)
Subject: [pypy-commit] pypy default: enable this to be called from somewhere
	else too
Message-ID: <20150409120228.85FDA1C0173@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r76753:4f4ec424db3b
Date: 2015-04-09 14:02 +0200
http://bitbucket.org/pypy/pypy/changeset/4f4ec424db3b/

Log:	enable this to be called from somewhere else too

diff --git a/rpython/translator/c/src/debug_print.c b/rpython/translator/c/src/debug_print.c
--- a/rpython/translator/c/src/debug_print.c
+++ b/rpython/translator/c/src/debug_print.c
@@ -26,9 +26,8 @@
 static char *debug_filename = NULL;
 static char *debug_filename_with_fork = NULL;
 
-static void pypy_debug_open(void)
+static void _pypy_debug_open(char *filename)
 {
-  char *filename = getenv("PYPYLOG");
   if (filename && filename[0])
     {
       char *colon = strchr(filename, ':');
@@ -77,6 +76,11 @@
   debug_ready = 1;
 }
 
+static void pypy_debug_open(void)
+{
+    _pypy_debug_open(getenv("PYPYLOG"));
+}
+
 long pypy_debug_offset(void)
 {
   if (!debug_ready)

From noreply at buildbot.pypy.org  Thu Apr  9 14:07:32 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Thu,  9 Apr 2015 14:07:32 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add test for an invariant that typeOf()
 and .convert_const() should obey
Message-ID: <20150409120732.915991C010B@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: 
Changeset: r76754:14453a4c1b2b
Date: 2015-04-09 13:07 +0100
http://bitbucket.org/pypy/pypy/changeset/14453a4c1b2b/

Log:	Add test for an invariant that typeOf() and .convert_const() should
	obey

diff --git a/rpython/rtyper/lltypesystem/test/test_lltype.py b/rpython/rtyper/lltypesystem/test/test_lltype.py
--- a/rpython/rtyper/lltypesystem/test/test_lltype.py
+++ b/rpython/rtyper/lltypesystem/test/test_lltype.py
@@ -1,9 +1,11 @@
-from __future__ import with_statement
 import py
+import sys
 from rpython.rtyper.lltypesystem.lltype import *
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.tool.identity_dict import identity_dict
 from rpython.tool import leakfinder
+from rpython.annotator.annrpython import RPythonAnnotator
+from rpython.rtyper.rtyper import RPythonTyper
 
 def isweak(p, T):
     try:
@@ -539,6 +541,24 @@
 
     assert S == Sprime
 
+class Frozen(object):
+    def _freeze_(self):
+        return True
+
+ at py.test.mark.parametrize('x', [
+    1, sys.maxint, 1.5, 'a', 'abc', u'abc', None, [],
+    py.test.mark.xfail(lambda: None),
+    {1.23: 'abc'},
+    (1, 'x', [2, 3.],),
+    py.test.mark.xfail(Frozen()),])
+def test_typeOf_const(x):
+    a = RPythonAnnotator()
+    bk = a.bookkeeper
+    rtyper = RPythonTyper(a)
+    s_x = bk.immutablevalue(x)
+    r_x = rtyper.getrepr(s_x)
+    assert typeOf(r_x.convert_const(x)) == r_x.lowleveltype
+
 def test_cast_primitive():
     cases = [
         (Float, 1, 1.0),

From noreply at buildbot.pypy.org  Thu Apr  9 14:08:42 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  9 Apr 2015 14:08:42 +0200 (CEST)
Subject: [pypy-commit] pypy default: pass jitdriver_sd instead of is_portal
	to jitcode
Message-ID: <20150409120842.DDA741C010B@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r76755:97dab258e764
Date: 2015-04-09 14:02 +0200
http://bitbucket.org/pypy/pypy/changeset/97dab258e764/

Log:	pass jitdriver_sd instead of is_portal to jitcode

diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -143,7 +143,7 @@
     def grab_initial_jitcodes(self):
         for jd in self.jitdrivers_sd:
             jd.mainjitcode = self.get_jitcode(jd.portal_graph)
-            jd.mainjitcode.is_portal = True
+            jd.mainjitcode.jitdriver_sd = jd
 
     def enum_pending_graphs(self):
         while self.unfinished_graphs:
diff --git a/rpython/jit/codewriter/jitcode.py b/rpython/jit/codewriter/jitcode.py
--- a/rpython/jit/codewriter/jitcode.py
+++ b/rpython/jit/codewriter/jitcode.py
@@ -12,7 +12,7 @@
         self.name = name
         self.fnaddr = fnaddr
         self.calldescr = calldescr
-        self.is_portal = False
+        self.jitdriver_sd = None # None for non-portals
         self._called_from = called_from   # debugging
         self._ssarepr     = None          # debugging
 
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -1624,7 +1624,7 @@
 
 def _handle_jitexception(blackholeinterp, exc):
     # See comments in _handle_jitexception_in_portal().
-    while not blackholeinterp.jitcode.is_portal:
+    while blackholeinterp.jitcode.jitdriver_sd is None:
         blackholeinterp.builder.release_interp(blackholeinterp)
         blackholeinterp = blackholeinterp.nextblackholeinterp
     if blackholeinterp.nextblackholeinterp is None:
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1778,7 +1778,7 @@
         return self.jitdriver_sd is not None and jitcode is self.jitdriver_sd.mainjitcode
 
     def newframe(self, jitcode, greenkey=None):
-        if jitcode.is_portal:
+        if jitcode.jitdriver_sd:
             self.portal_call_depth += 1
             self.call_ids.append(self.current_call_id)
             self.current_call_id += 1
@@ -1796,7 +1796,7 @@
     def popframe(self):
         frame = self.framestack.pop()
         jitcode = frame.jitcode
-        if jitcode.is_portal:
+        if jitcode.jitdriver_sd:
             self.portal_call_depth -= 1
             self.call_ids.pop()
         if frame.greenkey is not None and self.is_main_jitcode(jitcode):
@@ -1860,17 +1860,14 @@
         portal_call_depth = -1
         for frame in self.framestack:
             jitcode = frame.jitcode
-            assert jitcode.is_portal == len([
-                jd for jd in self.staticdata.jitdrivers_sd
-                   if jd.mainjitcode is jitcode])
-            if jitcode.is_portal:
+            if jitcode.jitdriver_sd:
                 portal_call_depth += 1
         if portal_call_depth != self.portal_call_depth:
             print "portal_call_depth problem!!!"
             print portal_call_depth, self.portal_call_depth
             for frame in self.framestack:
                 jitcode = frame.jitcode
-                if jitcode.is_portal:
+                if jitcode.jitdriver_sd:
                     print "P",
                 else:
                     print " ",
diff --git a/rpython/jit/metainterp/test/test_blackhole.py b/rpython/jit/metainterp/test/test_blackhole.py
--- a/rpython/jit/metainterp/test/test_blackhole.py
+++ b/rpython/jit/metainterp/test/test_blackhole.py
@@ -119,7 +119,7 @@
                       "\x01\x02",          # int_return/i
                       [],
                       num_regs_i=3, num_regs_r=0, num_regs_f=0)
-        jitcode.is_portal = True
+        jitcode.jitdriver_sd = "foo" # not none
         pc = 1
         registers_i = [history.BoxInt(40), history.ConstInt(2), None]
     class MyMetaInterp:

From noreply at buildbot.pypy.org  Thu Apr  9 14:08:44 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  9 Apr 2015 14:08:44 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20150409120844.218E01C010B@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r76756:196823d2c4e7
Date: 2015-04-09 14:08 +0200
http://bitbucket.org/pypy/pypy/changeset/196823d2c4e7/

Log:	merge

diff --git a/rpython/rtyper/lltypesystem/test/test_lltype.py b/rpython/rtyper/lltypesystem/test/test_lltype.py
--- a/rpython/rtyper/lltypesystem/test/test_lltype.py
+++ b/rpython/rtyper/lltypesystem/test/test_lltype.py
@@ -1,9 +1,11 @@
-from __future__ import with_statement
 import py
+import sys
 from rpython.rtyper.lltypesystem.lltype import *
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.tool.identity_dict import identity_dict
 from rpython.tool import leakfinder
+from rpython.annotator.annrpython import RPythonAnnotator
+from rpython.rtyper.rtyper import RPythonTyper
 
 def isweak(p, T):
     try:
@@ -539,6 +541,24 @@
 
     assert S == Sprime
 
+class Frozen(object):
+    def _freeze_(self):
+        return True
+
+ at py.test.mark.parametrize('x', [
+    1, sys.maxint, 1.5, 'a', 'abc', u'abc', None, [],
+    py.test.mark.xfail(lambda: None),
+    {1.23: 'abc'},
+    (1, 'x', [2, 3.],),
+    py.test.mark.xfail(Frozen()),])
+def test_typeOf_const(x):
+    a = RPythonAnnotator()
+    bk = a.bookkeeper
+    rtyper = RPythonTyper(a)
+    s_x = bk.immutablevalue(x)
+    r_x = rtyper.getrepr(s_x)
+    assert typeOf(r_x.convert_const(x)) == r_x.lowleveltype
+
 def test_cast_primitive():
     cases = [
         (Float, 1, 1.0),

From noreply at buildbot.pypy.org  Thu Apr  9 14:26:24 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  9 Apr 2015 14:26:24 +0200 (CEST)
Subject: [pypy-commit] pypy default: Test and fix for END_FINALLY's stack
	usage
Message-ID: <20150409122624.E5B1A1C010B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76757:e66f14f4e3d9
Date: 2015-04-09 14:26 +0200
http://bitbucket.org/pypy/pypy/changeset/e66f14f4e3d9/

Log:	Test and fix for END_FINALLY's stack usage

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -593,7 +593,9 @@
 
     ops.WITH_CLEANUP: -1,
     ops.POP_BLOCK: 0,
-    ops.END_FINALLY: -1,
+    ops.END_FINALLY: -3,     # assume always 3: we pretend that SETUP_FINALLY
+                             # pushes 3.  In truth, it would only push 1 and
+                             # the corresponding END_FINALLY only pops 1.
     ops.SETUP_WITH: 1,
     ops.SETUP_FINALLY: 0,
     ops.SETUP_EXCEPT: 0,
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -772,6 +772,24 @@
         code = compile_with_astcompiler(source, 'exec', self.space)
         assert code.co_stacksize == 2
 
+    def test_stackeffect_bug3(self):
+        source = """if 1:
+        try: pass
+        finally: pass
+        try: pass
+        finally: pass
+        try: pass
+        finally: pass
+        try: pass
+        finally: pass
+        try: pass
+        finally: pass
+        try: pass
+        finally: pass
+        """
+        code = compile_with_astcompiler(source, 'exec', self.space)
+        assert code.co_stacksize == 3
+
     def test_lambda(self):
         yield self.st, "y = lambda x: x", "y(4)", 4
 

From noreply at buildbot.pypy.org  Thu Apr  9 14:39:08 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  9 Apr 2015 14:39:08 +0200 (CEST)
Subject: [pypy-commit] pypy default: Test and fix
Message-ID: <20150409123908.9CDB91C00FA@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76758:72049e6e23d8
Date: 2015-04-09 14:37 +0200
http://bitbucket.org/pypy/pypy/changeset/72049e6e23d8/

Log:	Test and fix

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -406,7 +406,9 @@
                 elif (jump_op == ops.SETUP_FINALLY or
                       jump_op == ops.SETUP_EXCEPT or
                       jump_op == ops.SETUP_WITH):
-                    target_depth += 3
+                    if jump_op == ops.SETUP_WITH:
+                        target_depth -= 1     # ignore the w_result just pushed
+                    target_depth += 3         # add [exc_type, exc, unroller]
                     if target_depth > self._max_depth:
                         self._max_depth = target_depth
                 elif (jump_op == ops.JUMP_IF_TRUE_OR_POP or
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -790,6 +790,18 @@
         code = compile_with_astcompiler(source, 'exec', self.space)
         assert code.co_stacksize == 3
 
+    def test_stackeffect_bug4(self):
+        source = """if 1:
+        with a: pass
+        with a: pass
+        with a: pass
+        with a: pass
+        with a: pass
+        with a: pass
+        """
+        code = compile_with_astcompiler(source, 'exec', self.space)
+        assert code.co_stacksize == 4
+
     def test_lambda(self):
         yield self.st, "y = lambda x: x", "y(4)", 4
 

From noreply at buildbot.pypy.org  Thu Apr  9 14:45:09 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  9 Apr 2015 14:45:09 +0200 (CEST)
Subject: [pypy-commit] pypy default: Test and fix
Message-ID: <20150409124509.1271F1C116D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76759:d77902fffe58
Date: 2015-04-09 14:45 +0200
http://bitbucket.org/pypy/pypy/changeset/d77902fffe58/

Log:	Test and fix

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -565,10 +565,10 @@
     ops.INPLACE_OR: -1,
     ops.INPLACE_XOR: -1,
 
-    ops.SLICE+0: 1,
-    ops.SLICE+1: 0,
-    ops.SLICE+2: 0,
-    ops.SLICE+3: -1,
+    ops.SLICE+0: 0,
+    ops.SLICE+1: -1,
+    ops.SLICE+2: -1,
+    ops.SLICE+3: -2,
     ops.STORE_SLICE+0: -2,
     ops.STORE_SLICE+1: -3,
     ops.STORE_SLICE+2: -3,
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -802,6 +802,16 @@
         code = compile_with_astcompiler(source, 'exec', self.space)
         assert code.co_stacksize == 4
 
+    def test_stackeffect_bug5(self):
+        source = """if 1:
+        a[:]; a[:]; a[:]; a[:]; a[:]; a[:]
+        a[1:]; a[1:]; a[1:]; a[1:]; a[1:]; a[1:]
+        a[:2]; a[:2]; a[:2]; a[:2]; a[:2]; a[:2]
+        a[1:2]; a[1:2]; a[1:2]; a[1:2]; a[1:2]; a[1:2]
+        """
+        code = compile_with_astcompiler(source, 'exec', self.space)
+        assert code.co_stacksize == 3
+
     def test_lambda(self):
         yield self.st, "y = lambda x: x", "y(4)", 4
 

From noreply at buildbot.pypy.org  Thu Apr  9 14:52:52 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  9 Apr 2015 14:52:52 +0200 (CEST)
Subject: [pypy-commit] pypy default: Test and fix
Message-ID: <20150409125252.40F561C116D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76760:f57c8be4e18c
Date: 2015-04-09 14:52 +0200
http://bitbucket.org/pypy/pypy/changeset/f57c8be4e18c/

Log:	Test and fix

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -608,7 +608,6 @@
     ops.YIELD_VALUE: 0,
     ops.BUILD_CLASS: -2,
     ops.BUILD_MAP: 1,
-    ops.BUILD_SET: 1,
     ops.COMPARE_OP: -1,
 
     ops.LOOKUP_METHOD: 1,
@@ -663,6 +662,9 @@
 def _compute_BUILD_LIST(arg):
     return 1 - arg
 
+def _compute_BUILD_SET(arg):
+    return 1 - arg
+
 def _compute_MAKE_CLOSURE(arg):
     return -arg - 1
 
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -812,6 +812,13 @@
         code = compile_with_astcompiler(source, 'exec', self.space)
         assert code.co_stacksize == 3
 
+    def test_stackeffect_bug6(self):
+        source = """if 1:
+        {1}; {1}; {1}; {1}; {1}; {1}; {1}
+        """
+        code = compile_with_astcompiler(source, 'exec', self.space)
+        assert code.co_stacksize == 1
+
     def test_lambda(self):
         yield self.st, "y = lambda x: x", "y(4)", 4
 

From noreply at buildbot.pypy.org  Thu Apr  9 14:59:10 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  9 Apr 2015 14:59:10 +0200 (CEST)
Subject: [pypy-commit] pypy default: Check in the test that found the bugs
 in f57c8be4e18c, d77902fffe58, 
Message-ID: <20150409125910.31EF41C118B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76761:46d0755129c5
Date: 2015-04-09 14:59 +0200
http://bitbucket.org/pypy/pypy/changeset/46d0755129c5/

Log:	Check in the test that found the bugs in f57c8be4e18c, d77902fffe58,
	72049e6e23d8, e66f14f4e3d9.

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -9,6 +9,10 @@
 from pypy.tool import stdlib_opcode as ops
 
 
+class StackDepthComputationError(Exception):
+    pass
+
+
 class Instruction(object):
     """Represents a single opcode."""
 
@@ -55,11 +59,13 @@
     reaches the end of the block, it continues to next_block.
     """
 
+    marked = False
+    have_return = False
+    auto_inserted_return = False
+
     def __init__(self):
         self.instructions = []
         self.next_block = None
-        self.marked = False
-        self.have_return = False
 
     def _post_order_see(self, stack, nextblock):
         if nextblock.marked == 0:
@@ -384,7 +390,9 @@
         # look into a block when all the previous blocks have been done.
         self._max_depth = 0
         for block in blocks:
-            self._do_stack_depth_walk(block)
+            depth = self._do_stack_depth_walk(block)
+            if block.auto_inserted_return and depth != 0:
+                raise StackDepthComputationError   # fatal error
         return self._max_depth
 
     def _next_stack_depth_walk(self, nextblock, depth):
@@ -421,6 +429,7 @@
                     break
         if block.next_block and not done:
             self._next_stack_depth_walk(block.next_block, depth)
+        return depth
 
     def _build_lnotab(self, blocks):
         """Build the line number table for tracebacks and tracing."""
@@ -473,6 +482,7 @@
             if self.add_none_to_final_return:
                 self.load_const(self.space.w_None)
             self.emit_op(ops.RETURN_VALUE)
+            self.current_block.auto_inserted_return = True
         # Set the first lineno if it is not already explicitly set.
         if self.first_lineno == -1:
             if self.first_block.instructions:

From noreply at buildbot.pypy.org  Thu Apr  9 15:41:01 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  9 Apr 2015 15:41:01 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-locking: This is also a kind of lock...
Message-ID: <20150409134101.AC4C31C0173@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: c8-locking
Changeset: r1750:f7482306f0d4
Date: 2015-04-09 15:41 +0200
http://bitbucket.org/pypy/stmgc/changeset/f7482306f0d4/

Log:	This is also a kind of lock...

diff --git a/c8/LOCKS b/c8/LOCKS
--- a/c8/LOCKS
+++ b/c8/LOCKS
@@ -69,3 +69,13 @@
 The C_AT_SAFE_POINT and C_REQUEST_REMOVED condition codes are used by
 synchronize_all_threads().  That's used only in rare cases, for
 example because we want to start a major collection.
+
+
+
+usleep loop
+===========
+
+core.c: wait_for_other_inevitable()
+sync.c: stm_wait_for_current_inevitable_transaction()
+
+Must be fixed!

From noreply at buildbot.pypy.org  Thu Apr  9 15:58:40 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Thu,  9 Apr 2015 15:58:40 +0200 (CEST)
Subject: [pypy-commit] pypy object-dtype2: add tests that match -A behaviour,
 hack till they pass (cleanup needed)
Message-ID: <20150409135840.F0AB31C15B6@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: object-dtype2
Changeset: r76762:48af64138cf8
Date: 2015-04-09 10:51 +0300
http://bitbucket.org/pypy/pypy/changeset/48af64138cf8/

Log:	add tests that match -A behaviour, hack till they pass (cleanup
	needed)

diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -614,6 +614,8 @@
         self.w_obj = w_obj
 
     def convert_to(self, space, dtype):
+        if dtype.is_bool():
+            return W_BoolBox(space.bool_w(self.w_obj))
         return self # XXX
 
     def descr__getattr__(self, space, w_key):
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -202,11 +202,14 @@
             return self
         elif isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool() \
                 and w_idx.ndims() > 0:
-            return self.getitem_filter(space, w_idx)
+            w_res = self.getitem_filter(space, w_idx)
         try:
-            return self.implementation.descr_getitem(space, self, w_idx)
+            w_res = self.implementation.descr_getitem(space, self, w_idx)
         except ArrayArgumentException:
-            return self.getitem_array_int(space, w_idx)
+            w_res = self.getitem_array_int(space, w_idx)
+        if w_res.is_scalar() and w_res.get_dtype(space).is_object():
+            return w_res.get_dtype(space).itemtype.unbox(w_res)
+        return w_res
 
     def getitem(self, space, index_list):
         return self.implementation.getitem_index(space, index_list)
@@ -889,7 +892,10 @@
                 "The truth value of an array with more than one element "
                 "is ambiguous. Use a.any() or a.all()"))
         iter, state = self.create_iter()
-        return space.wrap(space.is_true(iter.getitem(state)))
+        w_val = iter.getitem(state)
+        if self.get_dtype().is_object():
+            w_val = self.get_dtype().itemtype.unbox(w_val) 
+        return space.wrap(space.is_true(w_val))
 
     def _binop_impl(ufunc_name):
         def impl(self, space, w_other, w_out=None):
diff --git a/pypy/module/micronumpy/test/test_object_arrays.py b/pypy/module/micronumpy/test/test_object_arrays.py
--- a/pypy/module/micronumpy/test/test_object_arrays.py
+++ b/pypy/module/micronumpy/test/test_object_arrays.py
@@ -33,16 +33,26 @@
         a = np.array(["foo"], dtype=object)
         b = a and complex(1, -1)
         assert b == complex(1, -1)
-        b = complex(1, -1) and a
+        b = np.array(complex(1, -1)) and a
         assert (b == a).all()
 
     def test_logical_ufunc(self):
         import numpy as np
         a = np.array(["foo"], dtype=object)
         b = np.array([1], dtype=object)
-        raises(TypeError, np.logical_and, a, 1)
-        raises(TypeError, np.logical_and, b, complex(1, -1))
+        d = np.array([complex(1, 10)], dtype=object)
+        c = np.logical_and(a, 1)
+        assert c.dtype == np.dtype('object')
+        assert c == 1
+        c = np.logical_and(b, complex(1, -1))
+        assert c.dtype == np.dtype('object')
+        assert c == complex(1, -1)
+        c = np.logical_and(d, b)
+        assert c == 1
         c = b & 1
+        assert c.dtype == np.dtype('object')
+        assert (c == 1).all()
+        c = np.array(1) & b
         assert (c == b).all()
 
     def test_reduce(self):
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -149,9 +149,8 @@
         return self.box(array[0])
 
     def unbox(self, box):
-        if not isinstance(box, self.BoxType):
-            # i.e. box is an ObjectBox
-            raise oefmt(self.space.w_AttributeError, '')
+        if isinstance(box, ObjectType.BoxType):
+            return box.w_obj
         return box.value
 
     def coerce(self, space, dtype, w_item):
@@ -1701,12 +1700,15 @@
 
     @specialize.argtype(1, 2)
     def box_complex(self, real, imag):
+        if isinstance(real, rffi.r_singlefloat):
+            real = rffi.cast(rffi.DOUBLE, real)
+        if isinstance(imag, rffi.r_singlefloat):
+            imag = rffi.cast(rffi.DOUBLE, imag)
         w_obj = self.space.newcomplex(real, imag)
         return self.BoxType(w_obj)
 
     def str_format(self, box):
-        return 'Object as string'
-        #return space.str_w(space.repr(self.unbox(box)))
+        return self.space.str_w(self.space.repr(self.unbox(box)))
 
     def to_builtin_type(self, space, box):
         assert isinstance(box, self.BoxType)
@@ -1716,7 +1718,7 @@
     def for_computation(v):
         return v
 
-    @raw_binary_op
+    @simple_binary_op
     def eq(self, v1, v2):
         return self.space.eq_w(v1, v2)
 
@@ -1735,28 +1737,32 @@
     def arctan2(self, v1, v2):
         raise oefmt(self.space.w_AttributeError, 'arctan2')
 
-    @raw_unary_op
+    @simple_unary_op
     def bool(self,v):
-        return not self.space.is_w(v, self.space.w_None) and \
-               not self.space.eq_w(v, self.space.wrap(0)) and \
-               not self.space.len_w(v) == 0 
+        return self._bool(v)
 
     def _bool(self, v):
-        return self.space.bool_w(v)
+        if self.space.is_true(v):
+            return True 
+        return False
 
-    @raw_binary_op
+    @simple_binary_op
     def logical_and(self, v1, v2):
-        return self._bool(v1) and self._bool(v2)
+        if self._bool(v1):
+            return v2
+        return v1
 
-    @raw_binary_op
+    @simple_binary_op
     def logical_or(self, v1, v2):
-        return self._bool(v1) or self._bool(v2)
+        if self._bool(v1):
+            return v1
+        return v2
 
-    @raw_unary_op
+    @simple_unary_op
     def logical_not(self, v):
         return not self._bool(v)
 
-    @raw_binary_op
+    @simple_binary_op
     def logical_xor(self, v1, v2):
         a = self._bool(v1)
         b = self._bool(v2)
@@ -1808,7 +1814,7 @@
     def func(self, w_v):
         w_impl = self.space.lookup(w_v, op)
         if w_impl is None:
-            raise oefmt(self.space.w_AttributeError, op)
+            raise oefmt(self.space.w_AttributeError, 'unknown op "%s" on object' % op)
         return self.space.get_and_call_function(w_impl, w_v)
     func.__name__ = 'object_' + op
     setattr(cls, op, func)
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -378,6 +378,8 @@
             w_val = self.func(calc_dtype,
                               w_obj.get_scalar_value().convert_to(space, calc_dtype))
             if out is None:
+                if res_dtype.is_object():
+                    w_val = w_obj.get_scalar_value()
                 return w_val
             w_val = res_dtype.coerce(space, w_val)
             if out.is_scalar():
@@ -481,7 +483,7 @@
         else:
             out = w_out
             calc_dtype = out.get_dtype()
-        if self.comparison_func:
+        if self.comparison_func and not calc_dtype.is_object():
             res_dtype = get_dtype_cache(space).w_booldtype
         else:
             res_dtype = calc_dtype
@@ -495,6 +497,8 @@
                     out.set_scalar_value(arr)
                 else:
                     out.fill(space, arr)
+            elif calc_dtype.is_object():
+                out = arr.get_scalar_value()
             else:
                 out = arr
             return out
@@ -1130,6 +1134,8 @@
         def impl(res_dtype, lvalue, rvalue):
             res = get_op(res_dtype)(lvalue, rvalue)
             if comparison_func:
+                if res_dtype.is_object():
+                    return res
                 return dtype_cache.w_booldtype.box(res)
             return res
     return func_with_new_name(impl, ufunc_name)

From noreply at buildbot.pypy.org  Thu Apr  9 15:58:42 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Thu,  9 Apr 2015 15:58:42 +0200 (CEST)
Subject: [pypy-commit] pypy object-dtype2: fix logic
Message-ID: <20150409135842.36D871C15B6@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: object-dtype2
Changeset: r76763:fc9ba3c58dda
Date: 2015-04-09 12:26 +0300
http://bitbucket.org/pypy/pypy/changeset/fc9ba3c58dda/

Log:	fix logic

diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py
--- a/pypy/module/micronumpy/base.py
+++ b/pypy/module/micronumpy/base.py
@@ -125,7 +125,7 @@
     def get_shape(self):
         return self.implementation.get_shape()
 
-    def get_dtype(self):
+    def get_dtype(self, space=None):
         return self.implementation.dtype
 
     def get_order(self):
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -203,10 +203,11 @@
         elif isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool() \
                 and w_idx.ndims() > 0:
             w_res = self.getitem_filter(space, w_idx)
-        try:
-            w_res = self.implementation.descr_getitem(space, self, w_idx)
-        except ArrayArgumentException:
-            w_res = self.getitem_array_int(space, w_idx)
+        else:
+            try:
+                w_res = self.implementation.descr_getitem(space, self, w_idx)
+            except ArrayArgumentException:
+                w_res = self.getitem_array_int(space, w_idx)
         if w_res.is_scalar() and w_res.get_dtype(space).is_object():
             return w_res.get_dtype(space).itemtype.unbox(w_res)
         return w_res

From noreply at buildbot.pypy.org  Thu Apr  9 15:58:43 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Thu,  9 Apr 2015 15:58:43 +0200 (CEST)
Subject: [pypy-commit] pypy object-dtype2: more tests,
	mimic numpy 1.9 ufunc behaviour
Message-ID: <20150409135843.70EEC1C15B6@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: object-dtype2
Changeset: r76764:dfa0d26cc06b
Date: 2015-04-09 16:03 +0300
http://bitbucket.org/pypy/pypy/changeset/dfa0d26cc06b/

Log:	more tests, mimic numpy 1.9 ufunc behaviour

diff --git a/pypy/module/micronumpy/test/test_object_arrays.py b/pypy/module/micronumpy/test/test_object_arrays.py
--- a/pypy/module/micronumpy/test/test_object_arrays.py
+++ b/pypy/module/micronumpy/test/test_object_arrays.py
@@ -66,6 +66,20 @@
         b = np.maximum.reduce(a)
         assert b is not None
 
+    def test_complex_op(self):
+        import numpy as np
+        a = np.array(['abc', 'def'], dtype=object) 
+        b = np.array([1, 2, 3], dtype=object) 
+        c = np.array([complex(1, 1), complex(1, -1)], dtype=object)
+        for arg in (a,b,c):
+            assert (arg == np.real(arg)).all()
+            assert (0 == np.imag(arg)).all()
+        raises(AttributeError, np.conj, a)
+        res = np.conj(b)
+        assert (res == b).all()
+        res = np.conj(c)
+        assert res[0] == c[1] and res[1] == c[0]
+
     def test_keep_object_alive(self):
         # only translated does it really test the gc
         import numpy as np
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -414,6 +414,9 @@
                         u(* [array] * u.nin)
                     except AttributeError:
                         pass
+                    except NotImplementedError:
+                        print s
+                        uncallable.add(s)
                     except TypeError:
                         assert s not in uncallable
                         uncallable.add(s)
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1734,9 +1734,6 @@
             return v1
         return v2
 
-    def arctan2(self, v1, v2):
-        raise oefmt(self.space.w_AttributeError, 'arctan2')
-
     @simple_unary_op
     def bool(self,v):
         return self._bool(v)
@@ -1800,22 +1797,39 @@
         else:
             return zero
 
+    @simple_unary_op
+    def real(self, v):
+        return v
 
+    @simple_unary_op
+    def imag(self, v):
+        return 0
+
+    @simple_unary_op
+    def square(self, v):
+        return self.space.mul(v, v)
+
+def add_attributeerr_op(cls, op):
+    def func(self, *args):
+        raise oefmt(self.space.w_AttributeError, 
+            "%s", op)
+    func.__name__ = 'object_' + op
+    setattr(cls, op, func)
+        
 def add_unsupported_op(cls, op):
     def func(self, *args):
         raise oefmt(self.space.w_TypeError, 
             "ufunc '%s' not supported for input types", op)
     func.__name__ = 'object_' + op
     setattr(cls, op, func)
-        
 
-def add_unary_op(cls, op):
+def add_unary_op(cls, op, method):
     @simple_unary_op
     def func(self, w_v):
-        w_impl = self.space.lookup(w_v, op)
+        w_impl = getattr(w_v, method, None)
         if w_impl is None:
             raise oefmt(self.space.w_AttributeError, 'unknown op "%s" on object' % op)
-        return self.space.get_and_call_function(w_impl, w_v)
+        return w_impl(self.space)
     func.__name__ = 'object_' + op
     setattr(cls, op, func)
 
@@ -1841,10 +1855,15 @@
     setattr(cls, op, func)
 
 for op in ('copysign', 'isfinite', 'isinf', 'isnan', 'logaddexp', 'logaddexp2',
-           'signbit',):
+           'signbit'):
     add_unsupported_op(ObjectType, op)
-for op in ('conj', 'real', 'imag', 'rint'):
-    add_unary_op(ObjectType, op)
+for op in ('arctan2', 'arccos', 'arccosh', 'arcsin', 'arcsinh', 'arctan',
+           'arctanh', 'ceil', 'floor', 'cos', 'sin', 'tan', 'cosh', 'sinh',
+           'tanh', 'radians', 'degrees', 'exp','exp2', 'expm1', 'fabs',
+           'log', 'log10', 'log1p', 'log2', 'sqrt', 'trunc'):
+    add_attributeerr_op(ObjectType, op)
+for op, method in (('conj', 'descr_conjugate'), ('rint', 'descr_rint')):
+    add_unary_op(ObjectType, op, method)
 for op in ('abs', 'neg', 'pos', 'invert'):
     add_space_unary_op(ObjectType, op)
 for op in ('add', 'floordiv', 'div', 'mod', 'mul', 'sub', 'lshift', 'rshift'):
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -497,10 +497,13 @@
                     out.set_scalar_value(arr)
                 else:
                     out.fill(space, arr)
-            elif calc_dtype.is_object():
-                out = arr.get_scalar_value()
+            elif isinstance(arr, res_dtype.itemtype.BoxType):
+                if res_dtype.is_object():
+                    out = arr.get_scalar_value()
+                else:
+                    out = arr
             else:
-                out = arr
+                out = space.wrap(arr)
             return out
         if isinstance(w_lhs, boxes.W_GenericBox):
             w_lhs = W_NDimArray.from_scalar(space, w_lhs)
@@ -1042,6 +1045,8 @@
 @jit.unroll_safe
 def find_unaryop_result_dtype(space, dt, promote_to_float=False,
         promote_bools=False, promote_to_largest=False):
+    if dt.is_object():
+        return dt
     if promote_to_largest:
         if dt.kind == NPY.GENBOOLLTR or dt.kind == NPY.SIGNEDLTR:
             if dt.elsize * 8 < LONG_BIT:
@@ -1128,6 +1133,9 @@
         def impl(res_dtype, value):
             res = get_op(res_dtype)(value)
             if bool_result:
+                if res_dtype.is_object() and isinstance(res, W_Root):
+                    space = res_dtype.itemtype.space
+                    res = space.bool_w(res)
                 return dtype_cache.w_booldtype.box(res)
             return res
     elif nin == 2:

From noreply at buildbot.pypy.org  Thu Apr  9 16:01:40 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  9 Apr 2015 16:01:40 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-locking: Two extra points (thanks remi)
Message-ID: <20150409140140.366EC1C15B6@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: c8-locking
Changeset: r1751:dc7653c95182
Date: 2015-04-09 16:02 +0200
http://bitbucket.org/pypy/stmgc/changeset/dc7653c95182/

Log:	Two extra points (thanks remi)

diff --git a/c8/LOCKS b/c8/LOCKS
--- a/c8/LOCKS
+++ b/c8/LOCKS
@@ -34,6 +34,12 @@
 the current segment modification_lock is also acquired briefly
 whenever we change our segment's 'modified_old_objects'.
 
+_validate_and_attach() needs to have its own segment's
+modification_lock *around* the compare-and-swap, so that
+_stm_validate() sees either the commit not done and the backup copies
+still in modified_old_objects, or the commit done and no backup copies
+any more.
+
 
 
 privatization_lock
@@ -70,6 +76,9 @@
 synchronize_all_threads().  That's used only in rare cases, for
 example because we want to start a major collection.
 
+The mutex also needs to be acquired for rewind_longjmp's setjmp() and
+longjmp() equivalent.
+
 
 
 usleep loop

From noreply at buildbot.pypy.org  Thu Apr  9 16:03:33 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Thu,  9 Apr 2015 16:03:33 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-hashtable: Close c8-hashtable branch ready
	to be merged
Message-ID: <20150409140333.D00101C1513@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-hashtable
Changeset: r1752:052ea809ba6f
Date: 2015-04-09 15:18 +0200
http://bitbucket.org/pypy/stmgc/changeset/052ea809ba6f/

Log:	Close c8-hashtable branch ready to be merged


From noreply at buildbot.pypy.org  Thu Apr  9 16:03:35 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Thu,  9 Apr 2015 16:03:35 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Merge with c8-hashtable
Message-ID: <20150409140335.1F5A91C1513@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: 
Changeset: r1753:a92ba2549987
Date: 2015-04-09 15:18 +0200
http://bitbucket.org/pypy/stmgc/changeset/a92ba2549987/

Log:	Merge with c8-hashtable

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -280,8 +280,14 @@
         struct stm_undo_s *end = undo + cl->written_count;
         for (; undo < end; undo++) {
             if (undo->type == TYPE_POSITION_MARKER) {
-                fprintf(stderr, "    marker %p %lu\n",
-                        undo->marker_object, undo->marker_odd_number);
+                if (undo->type2 == TYPE_MODIFIED_HASHTABLE) {
+                    fprintf(stderr, "    hashtable %p\n",
+                            undo->modif_hashtable);
+                }
+                else {
+                    fprintf(stderr, "    marker %p %lu\n",
+                            undo->marker_object, undo->marker_odd_number);
+                }
                 continue;
             }
             fprintf(stderr, "    obj %p, size %d, ofs %lu: ", undo->object,
@@ -383,21 +389,40 @@
                     struct stm_undo_s *undo = cl->written;
                     struct stm_undo_s *end = cl->written + cl->written_count;
                     for (; undo < end; undo++) {
-                        if (undo->type == TYPE_POSITION_MARKER)
+                        object_t *obj;
+
+                        if (undo->type != TYPE_POSITION_MARKER) {
+                            /* common case: 'undo->object' was written to
+                               in this past commit, so we must check that
+                               it was not read by us. */
+                            obj = undo->object;
+                        }
+                        else if (undo->type2 != TYPE_MODIFIED_HASHTABLE)
                             continue;
-                        if (_stm_was_read(undo->object)) {
-                            /* first reset all modified objects from the backup
-                               copies as soon as the first conflict is detected;
-                               then we will proceed below to update our segment from
-                               the old (but unmodified) version to the newer version.
-                            */
-                            reset_modified_from_backup_copies(my_segnum);
-                            timing_write_read_contention(cl->written, undo);
-                            needs_abort = true;
+                        else {
+                            /* the previous stm_undo_s is about a written
+                               'entry' object, which belongs to the hashtable
+                               given now.  Check that we haven't read the
+                               hashtable (via stm_hashtable_list()). */
+                            obj = undo->modif_hashtable;
+                        }
 
-                            dprintf(("_stm_validate() failed for obj %p\n", undo->object));
-                            break;
-                        }
+                        if (LIKELY(!_stm_was_read(obj)))
+                            continue;
+
+                        /* conflict! */
+                        dprintf(("_stm_validate() failed for obj %p\n", obj));
+
+                        /* first reset all modified objects from the backup
+                           copies as soon as the first conflict is detected;
+                           then we will proceed below to update our segment
+                           from the old (but unmodified) version to the newer
+                           version.
+                        */
+                        reset_modified_from_backup_copies(my_segnum);
+                        timing_write_read_contention(cl->written, undo);
+                        needs_abort = true;
+                        break;
                     }
                 }
 
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -190,9 +190,15 @@
         uintptr_t marker_odd_number; /* the odd number part of the marker */
         object_t *marker_object;     /* the object part of the marker */
     };
+    struct {
+        intptr_t type1;             /* TYPE_POSITION_MARKER (again) */
+        intptr_t type2;             /* TYPE_MODIFIED_HASHTABLE */
+        object_t *modif_hashtable;  /* modified entry is previous stm_undo_s */
+    };
   };
 };
 #define TYPE_POSITION_MARKER    (-1)
+#define TYPE_MODIFIED_HASHTABLE (-2)
 #define SLICE_OFFSET(slice)  ((slice) >> 16)
 #define SLICE_SIZE(slice)    ((int)((slice) & 0xFFFF))
 #define NEW_SLICE(offset, size) (((uint64_t)(offset)) << 16 | (size))
@@ -251,6 +257,14 @@
     return stm_object_pages + segment_num * (NB_PAGES * 4096UL);
 }
 
+static inline long get_num_segment_containing_address(char *addr)
+{
+    uintptr_t delta = addr - stm_object_pages;
+    uintptr_t result = delta / (NB_PAGES * 4096UL);
+    assert(result < NB_SEGMENTS);
+    return result;
+}
+
 static inline
 struct stm_segment_info_s *get_segment(long segment_num) {
     return (struct stm_segment_info_s *)REAL_ADDRESS(
@@ -285,6 +299,17 @@
 static void _signal_handler(int sig, siginfo_t *siginfo, void *context);
 static bool _stm_validate();
 
+static inline bool was_read_remote(char *base, object_t *obj)
+{
+    uint8_t other_transaction_read_version =
+        ((struct stm_segment_info_s *)REAL_ADDRESS(base, STM_PSEGMENT))
+            ->transaction_read_version;
+    uint8_t rm = ((struct stm_read_marker_s *)
+                  (base + (((uintptr_t)obj) >> 4)))->rm;
+    assert(rm <= other_transaction_read_version);
+    return rm == other_transaction_read_version;
+}
+
 static inline void _duck(void) {
     /* put a call to _duck() between two instructions that set 0 into
        a %gs-prefixed address and that may otherwise be replaced with
diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
--- a/c8/stm/gcpage.c
+++ b/c8/stm/gcpage.c
@@ -130,6 +130,58 @@
     return o;
 }
 
+static void _fill_preexisting_slice(long segnum, char *dest,
+                                    const char *src, uintptr_t size)
+{
+    uintptr_t np = dest - get_segment_base(segnum);
+    if (get_page_status_in(segnum, np / 4096) != PAGE_NO_ACCESS)
+        memcpy(dest, src, size);
+}
+
+object_t *stm_allocate_preexisting(ssize_t size_rounded_up,
+                                   const char *initial_data)
+{
+    stm_char *np = allocate_outside_nursery_large(size_rounded_up);
+    uintptr_t nobj = (uintptr_t)np;
+    dprintf(("allocate_preexisting: %p\n", (object_t *)nobj));
+
+    char *nobj_seg0 = stm_object_pages + nobj;
+    memcpy(nobj_seg0, initial_data, size_rounded_up);
+    ((struct object_s *)nobj_seg0)->stm_flags = GCFLAG_WRITE_BARRIER;
+
+    acquire_privatization_lock(STM_SEGMENT->segment_num);
+    DEBUG_EXPECT_SEGFAULT(false);
+
+    long j;
+    for (j = 1; j < NB_SEGMENTS; j++) {
+        const char *src = nobj_seg0;
+        char *dest = get_segment_base(j) + nobj;
+        char *end = dest + size_rounded_up;
+
+        while (((uintptr_t)dest) / 4096 != ((uintptr_t)end - 1) / 4096) {
+            uintptr_t count = 4096 - (((uintptr_t)dest) & 4095);
+            _fill_preexisting_slice(j, dest, src, count);
+            src += count;
+            dest += count;
+        }
+        _fill_preexisting_slice(j, dest, src, end - dest);
+
+#ifdef STM_TESTS
+        /* can't really enable this check outside tests, because there is
+           a change that the transaction_state changes in parallel */
+        if (get_priv_segment(j)->transaction_state != TS_NONE) {
+            assert(!was_read_remote(get_segment_base(j), (object_t *)nobj));
+        }
+#endif
+    }
+
+    DEBUG_EXPECT_SEGFAULT(true);
+    release_privatization_lock(STM_SEGMENT->segment_num);
+
+    write_fence();     /* make sure 'nobj' is fully initialized from
+                          all threads here */
+    return (object_t *)nobj;
+}
 
 /************************************************************/
 
@@ -249,6 +301,8 @@
 }
 
 
+#define TRACE_FOR_MAJOR_COLLECTION  (&mark_record_trace)
+
 static void mark_and_trace(
     object_t *obj,
     char *segment_base, /* to trace obj in */
@@ -408,7 +462,8 @@
         struct stm_undo_s *modified = (struct stm_undo_s *)lst->items;
         struct stm_undo_s *end = (struct stm_undo_s *)(lst->items + lst->count);
         for (; modified < end; modified++) {
-            if (modified->type == TYPE_POSITION_MARKER)
+            if (modified->type == TYPE_POSITION_MARKER &&
+                    modified->type2 != TYPE_MODIFIED_HASHTABLE)
                 mark_visit_possibly_new_object(modified->marker_object, pseg);
         }
     }
@@ -541,6 +596,31 @@
                 list_set_item(lst, n, list_pop_item(lst));
             }
         }
+
+        /* Remove from 'modified_old_objects' all old hashtables that die */
+        {
+            lst = pseg->modified_old_objects;
+            uintptr_t j, k = 0, limit = list_count(lst);
+            for (j = 0; j < limit; j += 3) {
+                uintptr_t e0 = list_item(lst, j + 0);
+                uintptr_t e1 = list_item(lst, j + 1);
+                uintptr_t e2 = list_item(lst, j + 2);
+                if (e0 == TYPE_POSITION_MARKER &&
+                    e1 == TYPE_MODIFIED_HASHTABLE &&
+                    !mark_visited_test((object_t *)e2)) {
+                    /* hashtable object dies */
+                }
+                else {
+                    if (j != k) {
+                        list_set_item(lst, k + 0, e0);
+                        list_set_item(lst, k + 1, e1);
+                        list_set_item(lst, k + 2, e2);
+                    }
+                    k += 3;
+                }
+            }
+            lst->count = k;
+        }
     }
 #pragma pop_macro("STM_SEGMENT")
 #pragma pop_macro("STM_PSEGMENT")
diff --git a/c8/stm/hashtable.c b/c8/stm/hashtable.c
new file mode 100644
--- /dev/null
+++ b/c8/stm/hashtable.c
@@ -0,0 +1,532 @@
+/*
+Design of stmgc's "hashtable" objects
+=====================================
+
+A "hashtable" is theoretically a lazily-filled array of objects of
+length 2**64.  Initially it is full of NULLs.  It's obviously
+implemented as a dictionary in which NULL objects are not needed.
+
+A real dictionary can be implemented on top of it, by using the index
+`hash(key)` in the hashtable, and storing a list of `(key, value)`
+pairs at that index (usually only one, unless there is a hash
+collision).
+
+The main operations on a hashtable are reading or writing an object at a
+given index.  It also supports fetching the list of non-NULL entries.
+
+There are two markers for every index (a read and a write marker).
+This is unlike regular arrays, which have only two markers in total.
+
+Additionally, we use the read marker for the hashtable object itself
+to mean "we have read the complete list of keys".  This plays the role
+of a "global" read marker: when any thread adds a new key/value object
+to the hashtable, this new object's read marker is initialized with a
+copy of the "global" read marker --- in all segments.
+
+
+Implementation
+--------------
+
+First idea: have the hashtable in raw memory, pointing to "entry"
+objects (which are regular, GC- and STM-managed objects).  The entry
+objects themselves point to the user-specified objects.  The entry
+objects hold the read/write markers.  Every entry object, once
+created, stays around.  It is only removed by the next major GC if it
+points to NULL and its read/write markers are not set in any
+currently-running transaction.
+
+References
+----------
+
+Inspired by: http://ppl.stanford.edu/papers/podc011-bronson.pdf
+*/
+
+
+uint32_t stm_hashtable_entry_userdata;
+
+
+#define INITIAL_HASHTABLE_SIZE   8
+#define PERTURB_SHIFT            5
+#define RESIZING_LOCK            0
+
+typedef struct {
+    uintptr_t mask;
+
+    /* 'resize_counter' start at an odd value, and is decremented (by
+       6) for every new item put in 'items'.  When it crosses 0, we
+       instead allocate a bigger table and change 'resize_counter' to
+       be a regular pointer to it (which is then even).  The whole
+       structure is immutable then.
+
+       The field 'resize_counter' also works as a write lock: changes
+       go via the intermediate value RESIZING_LOCK (0).
+    */
+    uintptr_t resize_counter;
+
+    stm_hashtable_entry_t *items[INITIAL_HASHTABLE_SIZE];
+} stm_hashtable_table_t;
+
+#define IS_EVEN(p) (((p) & 1) == 0)
+
+struct stm_hashtable_s {
+    stm_hashtable_table_t *table;
+    stm_hashtable_table_t initial_table;
+    uint64_t additions;
+};
+
+
+static inline void init_table(stm_hashtable_table_t *table, uintptr_t itemcount)
+{
+    table->mask = itemcount - 1;
+    table->resize_counter = itemcount * 4 + 1;
+    memset(table->items, 0, itemcount * sizeof(stm_hashtable_entry_t *));
+}
+
+stm_hashtable_t *stm_hashtable_create(void)
+{
+    stm_hashtable_t *hashtable = malloc(sizeof(stm_hashtable_t));
+    assert(hashtable);
+    hashtable->table = &hashtable->initial_table;
+    hashtable->additions = 0;
+    init_table(&hashtable->initial_table, INITIAL_HASHTABLE_SIZE);
+    return hashtable;
+}
+
+void stm_hashtable_free(stm_hashtable_t *hashtable)
+{
+    uintptr_t rc = hashtable->initial_table.resize_counter;
+    free(hashtable);
+    while (IS_EVEN(rc)) {
+        assert(rc != RESIZING_LOCK);
+
+        stm_hashtable_table_t *table = (stm_hashtable_table_t *)rc;
+        rc = table->resize_counter;
+        free(table);
+    }
+}
+
+static bool _stm_was_read_by_anybody(object_t *obj)
+{
+    /* can only be safely called during major GC, when all other threads
+       are suspended */
+    long i;
+    for (i = 1; i < NB_SEGMENTS; i++) {
+        if (get_priv_segment(i)->transaction_state == TS_NONE)
+            continue;
+        if (was_read_remote(get_segment_base(i), obj))
+            return true;
+    }
+    return false;
+}
+
+#define VOLATILE_HASHTABLE(p)    ((volatile stm_hashtable_t *)(p))
+#define VOLATILE_TABLE(p)  ((volatile stm_hashtable_table_t *)(p))
+
+static void _insert_clean(stm_hashtable_table_t *table,
+                          stm_hashtable_entry_t *entry,
+                          uintptr_t index)
+{
+    uintptr_t mask = table->mask;
+    uintptr_t i = index & mask;
+    if (table->items[i] == NULL) {
+        table->items[i] = entry;
+        return;
+    }
+
+    uintptr_t perturb = index;
+    while (1) {
+        i = (i << 2) + i + perturb + 1;
+        i &= mask;
+        if (table->items[i] == NULL) {
+            table->items[i] = entry;
+            return;
+        }
+
+        perturb >>= PERTURB_SHIFT;
+    }
+}
+
+static void _stm_rehash_hashtable(stm_hashtable_t *hashtable,
+                                  uintptr_t biggercount,
+                                  char *segment_base)
+{
+    dprintf(("rehash %p to size %ld, segment_base=%p\n",
+             hashtable, biggercount, segment_base));
+
+    size_t size = (offsetof(stm_hashtable_table_t, items)
+                   + biggercount * sizeof(stm_hashtable_entry_t *));
+    stm_hashtable_table_t *biggertable = malloc(size);
+    assert(biggertable);   // XXX
+
+    stm_hashtable_table_t *table = hashtable->table;
+    table->resize_counter = (uintptr_t)biggertable;
+    /* ^^^ this unlocks the table by writing a non-zero value to
+       table->resize_counter, but the new value is a pointer to the
+       new bigger table, so IS_EVEN() is still true */
+    assert(IS_EVEN(table->resize_counter));
+
+    init_table(biggertable, biggercount);
+
+    uintptr_t j, mask = table->mask;
+    uintptr_t rc = biggertable->resize_counter;
+    for (j = 0; j <= mask; j++) {
+        stm_hashtable_entry_t *entry = table->items[j];
+        if (entry == NULL)
+            continue;
+        if (segment_base != NULL) {
+            if (((struct stm_hashtable_entry_s *)
+                       REAL_ADDRESS(segment_base, entry))->object == NULL &&
+                   !_stm_was_read_by_anybody((object_t *)entry)) {
+                dprintf(("  removing dead %p\n", entry));
+                continue;
+            }
+        }
+
+        uintptr_t eindex;
+        if (segment_base == NULL)
+            eindex = entry->index;   /* read from STM_SEGMENT */
+        else
+            eindex = ((struct stm_hashtable_entry_s *)
+                       REAL_ADDRESS(segment_base, entry))->index;
+
+        dprintf(("  insert_clean %p at index=%ld\n",
+                 entry, eindex));
+        _insert_clean(biggertable, entry, eindex);
+        assert(rc > 6);
+        rc -= 6;
+    }
+    biggertable->resize_counter = rc;
+
+    write_fence();   /* make sure that 'biggertable' is valid here,
+                        and make sure 'table->resize_counter' is updated
+                        ('table' must be immutable from now on). */
+    VOLATILE_HASHTABLE(hashtable)->table = biggertable;
+}
+
+stm_hashtable_entry_t *stm_hashtable_lookup(object_t *hashtableobj,
+                                            stm_hashtable_t *hashtable,
+                                            uintptr_t index)
+{
+    stm_hashtable_table_t *table;
+    uintptr_t mask;
+    uintptr_t i;
+    stm_hashtable_entry_t *entry;
+
+ restart:
+    /* classical dict lookup logic */
+    table = VOLATILE_HASHTABLE(hashtable)->table;
+    mask = table->mask;      /* read-only field */
+    i = index & mask;
+    entry = VOLATILE_TABLE(table)->items[i];
+    if (entry != NULL) {
+        if (entry->index == index)
+            return entry;           /* found at the first try */
+
+        uintptr_t perturb = index;
+        while (1) {
+            i = (i << 2) + i + perturb + 1;
+            i &= mask;
+            entry = VOLATILE_TABLE(table)->items[i];
+            if (entry != NULL) {
+                if (entry->index == index)
+                    return entry;    /* found */
+            }
+            else
+                break;
+            perturb >>= PERTURB_SHIFT;
+        }
+    }
+    /* here, we didn't find the 'entry' with the correct index.  Note
+       that even if the same 'table' is modified or resized by other
+       threads concurrently, any new item found from a race condition
+       would anyway contain NULL in the present segment (ensured by
+       the first write_fence() below).  If the 'table' grows an entry
+       just after we checked above, then we go ahead and lock the
+       table; but after we get the lock, we will notice the new entry
+       (ensured by the second write_fence() below) and restart the
+       whole process.
+     */
+
+    uintptr_t rc = VOLATILE_TABLE(table)->resize_counter;
+
+    /* if rc is RESIZING_LOCK (which is 0, so even), a concurrent thread
+       is writing to the hashtable.  Or, if rc is another even number, it is
+       actually a pointer to the next version of the table, installed
+       just now.  In both cases, this thread must simply spin loop.
+    */
+    if (IS_EVEN(rc)) {
+        spin_loop();
+        goto restart;
+    }
+    /* in the other cases, we need to grab the RESIZING_LOCK.
+     */
+    if (!__sync_bool_compare_and_swap(&table->resize_counter,
+                                      rc, RESIZING_LOCK)) {
+        goto restart;
+    }
+    /* we now have the lock.  The only table with a non-even value of
+       'resize_counter' should be the last one in the chain, so if we
+       succeeded in locking it, check this. */
+    assert(table == hashtable->table);
+
+    /* Check that 'table->items[i]' is still NULL,
+       i.e. hasn't been populated under our feet.
+    */
+    if (table->items[i] != NULL) {
+        table->resize_counter = rc;    /* unlock */
+        goto restart;
+    }
+    /* if rc is greater than 6, there is enough room for a new
+       item in the current table.
+    */
+    if (rc > 6) {
+        /* we can only enter here once!  If we allocate stuff, we may
+           run the GC, and so 'hashtableobj' might move afterwards. */
+        if (_is_in_nursery(hashtableobj)) {
+            entry = (stm_hashtable_entry_t *)
+                stm_allocate(sizeof(stm_hashtable_entry_t));
+            entry->userdata = stm_hashtable_entry_userdata;
+            entry->index = index;
+            entry->object = NULL;
+        }
+        else {
+            /* for a non-nursery 'hashtableobj', we pretend that the
+               'entry' object we're about to return was already
+               existing all along, with NULL in all segments.  If the
+               caller of this function is going to modify the 'object'
+               field, it will call stm_write(entry) first, which will
+               correctly schedule 'entry' for write propagation.  We
+               do that even if 'hashtableobj' was created by the
+               running transaction: the new 'entry' object is created
+               as if it was older than the transaction.
+
+               Note the following difference: if 'hashtableobj' is
+               still in the nursery (case above), the 'entry' object
+               is also allocated from the nursery, and after a minor
+               collection it ages as an old-but-created-by-the-
+               current-transaction object.  We could try to emulate
+               this here, or to create young 'entry' objects, but
+               doing either of these would require careful
+               synchronization with other pieces of the code that may
+               change.
+            */
+            struct stm_hashtable_entry_s initial = {
+                .userdata = stm_hashtable_entry_userdata,
+                .index = index,
+                .object = NULL
+            };
+            entry = (stm_hashtable_entry_t *)
+                stm_allocate_preexisting(sizeof(stm_hashtable_entry_t),
+                                         (char *)&initial.header);
+            hashtable->additions++;
+        }
+        table->items[i] = entry;
+        write_fence();     /* make sure 'table->items' is written here */
+        VOLATILE_TABLE(table)->resize_counter = rc - 6;    /* unlock */
+        return entry;
+    }
+    else {
+        /* if rc is smaller than 6, we must allocate a new bigger table.
+         */
+        uintptr_t biggercount = table->mask + 1;
+        if (biggercount < 50000)
+            biggercount *= 4;
+        else
+            biggercount *= 2;
+        _stm_rehash_hashtable(hashtable, biggercount, /*segment_base=*/NULL);
+        goto restart;
+    }
+}
+
+object_t *stm_hashtable_read(object_t *hobj, stm_hashtable_t *hashtable,
+                             uintptr_t key)
+{
+    stm_hashtable_entry_t *e = stm_hashtable_lookup(hobj, hashtable, key);
+    stm_read((object_t *)e);
+    return e->object;
+}
+
+void stm_hashtable_write_entry(object_t *hobj, stm_hashtable_entry_t *entry,
+                               object_t *nvalue)
+{
+    if (_STM_WRITE_CHECK_SLOWPATH((object_t *)entry)) {
+
+        stm_write((object_t *)entry);
+
+        uintptr_t i = list_count(STM_PSEGMENT->modified_old_objects);
+        if (i > 0 && list_item(STM_PSEGMENT->modified_old_objects, i - 3)
+                     == (uintptr_t)entry) {
+            /* The stm_write() above recorded a write to 'entry'.  Here,
+               we add another stm_undo_s to modified_old_objects with
+               TYPE_MODIFIED_HASHTABLE.  It is ignored everywhere except
+               in _stm_validate().
+
+               The goal is that this TYPE_MODIFIED_HASHTABLE ends up in
+               the commit log's 'cl_written' array.  Later, another
+               transaction validating that log will check two things:
+
+               - the regular stm_undo_s entry put by stm_write() above
+                 will make the other transaction check that it didn't
+                 read the same 'entry' object;
+
+                 - the TYPE_MODIFIED_HASHTABLE entry we're adding now
+                   will make the other transaction check that it didn't
+                   do any stm_hashtable_list() on the complete hashtable.
+            */
+            STM_PSEGMENT->modified_old_objects = list_append3(
+                STM_PSEGMENT->modified_old_objects,
+                TYPE_POSITION_MARKER,      /* type1 */
+                TYPE_MODIFIED_HASHTABLE,   /* type2 */
+                (uintptr_t)hobj);          /* modif_hashtable */
+        }
+    }
+    entry->object = nvalue;
+}
+
+void stm_hashtable_write(object_t *hobj, stm_hashtable_t *hashtable,
+                         uintptr_t key, object_t *nvalue,
+                         stm_thread_local_t *tl)
+{
+    STM_PUSH_ROOT(*tl, nvalue);
+    STM_PUSH_ROOT(*tl, hobj);
+    stm_hashtable_entry_t *e = stm_hashtable_lookup(hobj, hashtable, key);
+    STM_POP_ROOT(*tl, hobj);
+    STM_POP_ROOT(*tl, nvalue);
+    stm_hashtable_write_entry(hobj, e, nvalue);
+}
+
+long stm_hashtable_length_upper_bound(stm_hashtable_t *hashtable)
+{
+    stm_hashtable_table_t *table;
+    uintptr_t rc;
+
+ restart:
+    table = VOLATILE_HASHTABLE(hashtable)->table;
+    rc = VOLATILE_TABLE(table)->resize_counter;
+    if (IS_EVEN(rc)) {
+        spin_loop();
+        goto restart;
+    }
+
+    uintptr_t initial_rc = (table->mask + 1) * 4 + 1;
+    uintptr_t num_entries_times_6 = initial_rc - rc;
+    return num_entries_times_6 / 6;
+}
+
+long stm_hashtable_list(object_t *hobj, stm_hashtable_t *hashtable,
+                        stm_hashtable_entry_t **results)
+{
+    /* Set the read marker.  It will be left as long as we're running
+       the same transaction.
+    */
+    stm_read(hobj);
+
+    /* Get the table.  No synchronization is needed: we may miss some
+       entries that are being added, but they would contain NULL in
+       this segment anyway. */
+    stm_hashtable_table_t *table = VOLATILE_HASHTABLE(hashtable)->table;
+
+    /* Read all entries, check which ones are not NULL, count them,
+       and optionally list them in 'results'.
+    */
+    uintptr_t i, mask = table->mask;
+    stm_hashtable_entry_t *entry;
+    long nresult = 0;
+
+    if (results != NULL) {
+        /* collect the results in the provided list */
+        for (i = 0; i <= mask; i++) {
+            entry = VOLATILE_TABLE(table)->items[i];
+            if (entry != NULL) {
+                stm_read((object_t *)entry);
+                if (entry->object != NULL)
+                    results[nresult++] = entry;
+            }
+        }
+    }
+    else {
+        /* don't collect, just get the exact number of results */
+        for (i = 0; i <= mask; i++) {
+            entry = VOLATILE_TABLE(table)->items[i];
+            if (entry != NULL) {
+                stm_read((object_t *)entry);
+                if (entry->object != NULL)
+                    nresult++;
+            }
+        }
+    }
+    return nresult;
+}
+
+static void _stm_compact_hashtable(struct object_s *hobj,
+                                   stm_hashtable_t *hashtable)
+{
+    stm_hashtable_table_t *table = hashtable->table;
+    uintptr_t rc = table->resize_counter;
+    assert(!IS_EVEN(rc));
+
+    if (hashtable->additions * 4 > table->mask) {
+        hashtable->additions = 0;
+
+        /* If 'hobj' was created in some current transaction, i.e. if it is
+           now an overflow object, then we have the risk that some of its
+           entry objects were not created with stm_allocate_preexisting().
+           In that situation, a valid workaround is to read all entry
+           objects in the segment of the running transaction.  Otherwise,
+           the base case is to read them all from segment zero.
+        */
+        long segnum = get_num_segment_containing_address((char *)hobj);
+        if (!IS_OVERFLOW_OBJ(get_priv_segment(segnum), hobj))
+            segnum = 0;
+
+        uintptr_t initial_rc = (table->mask + 1) * 4 + 1;
+        uintptr_t num_entries_times_6 = initial_rc - rc;
+        uintptr_t count = INITIAL_HASHTABLE_SIZE;
+        while (count * 4 < num_entries_times_6)
+            count *= 2;
+        /* sanity-check: 'num_entries_times_6 < initial_rc', and so 'count'
+           can never grow larger than the current table size. */
+        assert(count <= table->mask + 1);
+
+        dprintf(("compact with %ld items:\n", num_entries_times_6 / 6));
+        _stm_rehash_hashtable(hashtable, count, get_segment_base(segnum));
+    }
+
+    table = hashtable->table;
+    assert(!IS_EVEN(table->resize_counter));
+
+    if (table != &hashtable->initial_table) {
+        uintptr_t rc = hashtable->initial_table.resize_counter;
+        while (1) {
+            assert(IS_EVEN(rc));
+            assert(rc != RESIZING_LOCK);
+
+            stm_hashtable_table_t *old_table = (stm_hashtable_table_t *)rc;
+            if (old_table == table)
+                break;
+            rc = old_table->resize_counter;
+            free(old_table);
+        }
+        hashtable->initial_table.resize_counter = (uintptr_t)table;
+        assert(IS_EVEN(hashtable->initial_table.resize_counter));
+    }
+}
+
+void stm_hashtable_tracefn(struct object_s *hobj, stm_hashtable_t *hashtable,
+                           void trace(object_t **))
+{
+    if (trace == TRACE_FOR_MAJOR_COLLECTION)
+        _stm_compact_hashtable(hobj, hashtable);
+
+    stm_hashtable_table_t *table;
+    table = VOLATILE_HASHTABLE(hashtable)->table;
+
+    uintptr_t j, mask = table->mask;
+    for (j = 0; j <= mask; j++) {
+        stm_hashtable_entry_t *volatile *pentry;
+        pentry = &VOLATILE_TABLE(table)->items[j];
+        if (*pentry != NULL) {
+            trace((object_t **)pentry);
+        }
+    }
+}
diff --git a/c8/stm/marker.c b/c8/stm/marker.c
--- a/c8/stm/marker.c
+++ b/c8/stm/marker.c
@@ -42,7 +42,8 @@
     */
     while (contention != start) {
         --contention;
-        if (contention->type == TYPE_POSITION_MARKER) {
+        if (contention->type == TYPE_POSITION_MARKER &&
+            contention->type2 != TYPE_MODIFIED_HASHTABLE) {
             out_marker->odd_number = contention->marker_odd_number;
             out_marker->object = contention->marker_object;
             return;
@@ -69,6 +70,9 @@
             return;    /* already up-to-date */
     }
 
+    /* -2 is not odd */
+    assert(marker.odd_number != (uintptr_t)TYPE_MODIFIED_HASHTABLE);
+
     STM_PSEGMENT->position_markers_last = list_count(list);
     STM_PSEGMENT->modified_old_objects = list_append3(
         list,
diff --git a/c8/stm/misc.c b/c8/stm/misc.c
--- a/c8/stm/misc.c
+++ b/c8/stm/misc.c
@@ -31,10 +31,7 @@
 
 bool _stm_was_read(object_t *obj)
 {
-    uint8_t rm = ((struct stm_read_marker_s *)
-                  (STM_SEGMENT->segment_base + (((uintptr_t)obj) >> 4)))->rm;
-    assert(rm <= STM_SEGMENT->transaction_read_version);
-    return rm == STM_SEGMENT->transaction_read_version;
+    return was_read_remote(STM_SEGMENT->segment_base, obj);
 }
 
 bool _stm_was_written(object_t *obj)
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -424,6 +424,7 @@
     struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count);
 
     for (; undo < end; undo++) {
+        /* this logic also works if type2 == TYPE_MODIFIED_HASHTABLE */
         if (undo->type == TYPE_POSITION_MARKER)
             minor_trace_if_young(&undo->marker_object);
     }
diff --git a/c8/stm/pages.h b/c8/stm/pages.h
--- a/c8/stm/pages.h
+++ b/c8/stm/pages.h
@@ -62,7 +62,11 @@
 
 static inline bool get_page_status_in(long segnum, uintptr_t pagenum)
 {
-    /* reading page status requires "read"-lock: */
+    /* reading page status requires "read"-lock, which is defined as
+       "any segment has the privatization_lock".  This is enough to
+       prevent the "write"-lock from being acquired by somebody else
+       (defined as "_all_ segments have the privatization_lock").
+    */
     assert(STM_PSEGMENT->privatization_lock);
 
     OPT_ASSERT(segnum < 8 * sizeof(struct page_shared_s));
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -250,8 +250,6 @@
     set_gs_register(get_segment_base(num + 1));
     s_mutex_unlock();
 
-    DEBUG_EXPECT_SEGFAULT(true);
-
     if (num == 0) {
         dprintf(("STM_GC_NURSERY: %d\n", STM_GC_NURSERY));
         dprintf(("NB_PAGES: %d\n", NB_PAGES));
diff --git a/c8/stm/setup.h b/c8/stm/setup.h
--- a/c8/stm/setup.h
+++ b/c8/stm/setup.h
@@ -3,8 +3,8 @@
 static pthread_t *_get_cpth(stm_thread_local_t *);
 
 #ifndef NDEBUG
-static __thread long _stm_segfault_expected = 0;
-#define DEBUG_EXPECT_SEGFAULT(v) do {if (v) _stm_segfault_expected++; else _stm_segfault_expected--;} while (0)
+static __thread long _stm_segfault_expected = 1;
+#define DEBUG_EXPECT_SEGFAULT(v) do {if (v) _stm_segfault_expected++; else _stm_segfault_expected--; assert(_stm_segfault_expected <= 1);} while (0)
 #else
 #define DEBUG_EXPECT_SEGFAULT(v) {}
 #endif
diff --git a/c8/stmgc.c b/c8/stmgc.c
--- a/c8/stmgc.c
+++ b/c8/stmgc.c
@@ -39,3 +39,4 @@
 #include "stm/prof.c"
 #include "stm/rewind_setjmp.c"
 #include "stm/finalizer.c"
+#include "stm/hashtable.c"
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -196,10 +196,13 @@
         STM_SEGMENT->transaction_read_version;
 }
 
+#define _STM_WRITE_CHECK_SLOWPATH(obj)  \
+    UNLIKELY(((obj)->stm_flags & _STM_GCFLAG_WRITE_BARRIER) != 0)
+
 __attribute__((always_inline))
 static inline void stm_write(object_t *obj)
 {
-    if (UNLIKELY((obj->stm_flags & _STM_GCFLAG_WRITE_BARRIER) != 0))
+    if (_STM_WRITE_CHECK_SLOWPATH(obj))
         _stm_write_slowpath(obj);
 }
 
@@ -208,7 +211,7 @@
 static inline void stm_write_card(object_t *obj, uintptr_t index)
 {
     /* if GCFLAG_WRITE_BARRIER is set, then don't do anything more. */
-    if (UNLIKELY((obj->stm_flags & _STM_GCFLAG_WRITE_BARRIER) != 0)) {
+    if (_STM_WRITE_CHECK_SLOWPATH(obj)) {
 
         /* GCFLAG_WRITE_BARRIER is not set.  This might be because
            it's the first time we see a given small array; or it might
@@ -479,6 +482,38 @@
 /* dummies for now: */
 static inline void stm_flush_timing(stm_thread_local_t *tl, int verbose) {}
 
+
+/* Hashtables.  Keys are 64-bit unsigned integers, values are
+   'object_t *'.  Note that the type 'stm_hashtable_t' is not an
+   object type at all; you need to allocate and free it explicitly.
+   If you want to embed the hashtable inside an 'object_t' you
+   probably need a light finalizer to do the freeing. */
+typedef struct stm_hashtable_s stm_hashtable_t;
+typedef TLPREFIX struct stm_hashtable_entry_s stm_hashtable_entry_t;
+
+stm_hashtable_t *stm_hashtable_create(void);
+void stm_hashtable_free(stm_hashtable_t *);
+stm_hashtable_entry_t *stm_hashtable_lookup(object_t *, stm_hashtable_t *,
+                                            uintptr_t key);
+object_t *stm_hashtable_read(object_t *, stm_hashtable_t *, uintptr_t key);
+void stm_hashtable_write(object_t *, stm_hashtable_t *, uintptr_t key,
+                         object_t *nvalue, stm_thread_local_t *);
+void stm_hashtable_write_entry(object_t *hobj, stm_hashtable_entry_t *entry,
+                               object_t *nvalue);
+long stm_hashtable_length_upper_bound(stm_hashtable_t *);
+long stm_hashtable_list(object_t *, stm_hashtable_t *,
+                        stm_hashtable_entry_t **results);
+extern uint32_t stm_hashtable_entry_userdata;
+void stm_hashtable_tracefn(struct object_s *, stm_hashtable_t *,
+                           void (object_t **));
+
+struct stm_hashtable_entry_s {
+    struct object_s header;
+    uint32_t userdata;
+    uintptr_t index;
+    object_t *object;
+};
+
 /* ==================== END ==================== */
 
 static void (*stmcb_expand_marker)(char *segment_base, uintptr_t odd_number,
diff --git a/c8/test/support.py b/c8/test/support.py
--- a/c8/test/support.py
+++ b/c8/test/support.py
@@ -197,6 +197,26 @@
 void stm_enable_light_finalizer(object_t *);
 
 void (*stmcb_finalizer)(object_t *);
+
+typedef struct stm_hashtable_s stm_hashtable_t;
+typedef ... stm_hashtable_entry_t;
+stm_hashtable_t *stm_hashtable_create(void);
+void stm_hashtable_free(stm_hashtable_t *);
+bool _check_hashtable_read(object_t *, stm_hashtable_t *, uintptr_t key);
+object_t *hashtable_read_result;
+bool _check_hashtable_write(object_t *, stm_hashtable_t *, uintptr_t key,
+                            object_t *nvalue, stm_thread_local_t *tl);
+long stm_hashtable_length_upper_bound(stm_hashtable_t *);
+long stm_hashtable_list(object_t *, stm_hashtable_t *,
+                        stm_hashtable_entry_t **results);
+uint32_t stm_hashtable_entry_userdata;
+void stm_hashtable_tracefn(struct object_s *, stm_hashtable_t *,
+                           void trace(object_t **));
+
+void _set_hashtable(object_t *obj, stm_hashtable_t *h);
+stm_hashtable_t *_get_hashtable(object_t *obj);
+uintptr_t _get_entry_index(stm_hashtable_entry_t *entry);
+object_t *_get_entry_object(stm_hashtable_entry_t *entry);
 """)
 
 
@@ -299,6 +319,19 @@
     CHECKED(stm_validate());
 }
 
+object_t *hashtable_read_result;
+
+bool _check_hashtable_read(object_t *hobj, stm_hashtable_t *h, uintptr_t key)
+{
+    CHECKED(hashtable_read_result = stm_hashtable_read(hobj, h, key));
+}
+
+bool _check_hashtable_write(object_t *hobj, stm_hashtable_t *h, uintptr_t key,
+                            object_t *nvalue, stm_thread_local_t *tl)
+{
+    CHECKED(stm_hashtable_write(hobj, h, key, nvalue, tl));
+}
+
 #undef CHECKED
 
 
@@ -326,6 +359,32 @@
     return *WEAKREF_PTR(obj, size);
 }
 
+void _set_hashtable(object_t *obj, stm_hashtable_t *h)
+{
+    stm_char *field_addr = ((stm_char*)obj);
+    field_addr += SIZEOF_MYOBJ; /* header */
+    *(stm_hashtable_t *TLPREFIX *)field_addr = h;
+}
+
+stm_hashtable_t *_get_hashtable(object_t *obj)
+{
+    stm_char *field_addr = ((stm_char*)obj);
+    field_addr += SIZEOF_MYOBJ; /* header */
+    return *(stm_hashtable_t *TLPREFIX *)field_addr;
+}
+
+uintptr_t _get_entry_index(stm_hashtable_entry_t *entry)
+{
+    stm_read((object_t *)entry);
+    return entry->index;
+}
+
+object_t *_get_entry_object(stm_hashtable_entry_t *entry)
+{
+    stm_read((object_t *)entry);
+    return entry->object;
+}
+
 void _set_ptr(object_t *obj, int n, object_t *v)
 {
     long nrefs = (long)((myobj_t*)obj)->type_id - 421420;
@@ -351,11 +410,17 @@
 }
 
 
-
 ssize_t stmcb_size_rounded_up(struct object_s *obj)
 {
     struct myobj_s *myobj = (struct myobj_s*)obj;
+    assert(myobj->type_id != 0);
     if (myobj->type_id < 421420) {
+        if (myobj->type_id == 421419) {    /* hashtable */
+            return sizeof(struct myobj_s) + 1 * sizeof(void*);
+        }
+        if (myobj->type_id == 421418) {    /* hashtable entry */
+            return sizeof(struct stm_hashtable_entry_s);
+        }
         /* basic case: tid equals 42 plus the size of the object */
         assert(myobj->type_id >= 42 + sizeof(struct myobj_s));
         assert((myobj->type_id - 42) >= 16);
@@ -371,11 +436,21 @@
     }
 }
 
-
 void stmcb_trace(struct object_s *obj, void visit(object_t **))
 {
     int i;
     struct myobj_s *myobj = (struct myobj_s*)obj;
+    if (myobj->type_id == 421419) {
+        /* hashtable */
+        stm_hashtable_t *h = *((stm_hashtable_t **)(myobj + 1));
+        stm_hashtable_tracefn(obj, h, visit);
+        return;
+    }
+    if (myobj->type_id == 421418) {
+        /* hashtable entry */
+        object_t **ref = &((struct stm_hashtable_entry_s *)myobj)->object;
+        visit(ref);
+    }
     if (myobj->type_id < 421420) {
         /* basic case: no references */
         return;
@@ -396,6 +471,7 @@
 {
     int i;
     struct myobj_s *myobj = (struct myobj_s*)obj;
+    assert(myobj->type_id != 0);
     assert(myobj->type_id != 421419);
     assert(myobj->type_id != 421418);
     if (myobj->type_id < 421420) {
@@ -413,6 +489,9 @@
                                   uintptr_t offset_itemsize[2])
 {
     struct myobj_s *myobj = (struct myobj_s*)obj;
+    assert(myobj->type_id != 0);
+    assert(myobj->type_id != 421419);
+    assert(myobj->type_id != 421418);
     if (myobj->type_id < 421420) {
         offset_itemsize[0] = SIZEOF_MYOBJ;
         offset_itemsize[1] = 1;
@@ -468,6 +547,7 @@
 CARD_CLEAR = 0
 CARD_MARKED = lib._STM_CARD_MARKED
 CARD_MARKED_OLD = lib._stm_get_transaction_read_version
+lib.stm_hashtable_entry_userdata = 421418
 
 
 class Conflict(Exception):
@@ -530,6 +610,18 @@
     lib._set_type_id(o, tid)
     return o
 
+def stm_allocate_hashtable():
+    o = lib.stm_allocate(16)
+    tid = 421419
+    lib._set_type_id(o, tid)
+    h = lib.stm_hashtable_create()
+    lib._set_hashtable(o, h)
+    return o
+
+def get_hashtable(o):
+    assert lib._get_type_id(o) == 421419
+    return lib._get_hashtable(o)
+
 def stm_get_weakref(o):
     return lib._get_weakref(o)
 
diff --git a/c8/test/test_hashtable.py b/c8/test/test_hashtable.py
new file mode 100644
--- /dev/null
+++ b/c8/test/test_hashtable.py
@@ -0,0 +1,561 @@
+from support import *
+import random
+import py, sys
+
+
+def htget(o, key):
+    h = get_hashtable(o)
+    res = lib._check_hashtable_read(o, h, key)
+    if res:
+        raise Conflict
+    return lib.hashtable_read_result
+
+def htset(o, key, nvalue, tl):
+    h = get_hashtable(o)
+    res = lib._check_hashtable_write(o, h, key, nvalue, tl)
+    if res:
+        raise Conflict
+
+def ht_length_upper_bound(o):
+    h = get_hashtable(o)
+    return lib.stm_hashtable_length_upper_bound(h)
+
+def htitems(o):
+    h = get_hashtable(o)
+    upper_bound = lib.stm_hashtable_length_upper_bound(h)
+    entries = ffi.new("stm_hashtable_entry_t *[]", upper_bound)
+    count = lib.stm_hashtable_list(o, h, entries)
+    assert count <= upper_bound
+    return [(lib._get_entry_index(entries[i]),
+             lib._get_entry_object(entries[i])) for i in range(count)]
+
+def htlen(o):
+    h = get_hashtable(o)
+    count = lib.stm_hashtable_list(o, h, ffi.NULL)
+    return count
+
+
+class BaseTestHashtable(BaseTest):
+
+    def setup_method(self, meth):
+        BaseTest.setup_method(self, meth)
+        #
+        @ffi.callback("void(object_t *)")
+        def light_finalizer(obj):
+            print 'light_finalizer:', obj
+            try:
+                assert lib._get_type_id(obj) == 421419
+                self.seen_hashtables -= 1
+            except:
+                self.errors.append(sys.exc_info()[2])
+                raise
+
+        lib.stmcb_light_finalizer = light_finalizer
+        self._light_finalizer_keepalive = light_finalizer
+        self.seen_hashtables = 0
+        self.errors = []
+
+    def teardown_method(self, meth):
+        BaseTest.teardown_method(self, meth)
+        lib.stmcb_light_finalizer = ffi.NULL
+        assert self.errors == []
+        assert self.seen_hashtables == 0
+
+    def allocate_hashtable(self):
+        h = stm_allocate_hashtable()
+        lib.stm_enable_light_finalizer(h)
+        self.seen_hashtables += 1
+        return h
+
+
+class TestHashtable(BaseTestHashtable):
+
+    def test_empty(self):
+        self.start_transaction()
+        h = self.allocate_hashtable()
+        for i in range(100):
+            index = random.randrange(0, 1<<64)
+            got = htget(h, index)
+            assert got == ffi.NULL
+
+    def test_set_value(self):
+        self.start_transaction()
+        tl0 = self.tls[self.current_thread]
+        h = self.allocate_hashtable()
+        lp1 = stm_allocate(16)
+        htset(h, 12345678901, lp1, tl0)
+        assert htget(h, 12345678901) == lp1
+        for i in range(64):
+            index = 12345678901 ^ (1 << i)
+            assert htget(h, index) == ffi.NULL
+        assert htget(h, 12345678901) == lp1
+
+    def test_no_conflict(self):
+        lp1 = stm_allocate_old(16)
+        lp2 = stm_allocate_old(16)
+        #
+        self.start_transaction()
+        tl0 = self.tls[self.current_thread]
+        h = self.allocate_hashtable()
+        self.push_root(h)
+        stm_set_char(lp1, 'A')
+        htset(h, 1234, lp1, tl0)
+        self.commit_transaction()
+        #
+        self.start_transaction()
+        h = self.pop_root()
+        stm_set_char(lp2, 'B')
+        self.switch(1)
+        self.start_transaction()
+        self.switch(0)
+        htset(h, 9991234, lp2, tl0)
+        #
+        self.switch(1)
+        lp1b = htget(h, 1234)
+        assert lp1b != ffi.NULL
+        assert stm_get_char(lp1b) == 'A'
+        assert lp1b == lp1
+        self.commit_transaction()
+        #
+        self.switch(0)
+        assert htget(h, 9991234) == lp2
+        assert stm_get_char(lp2) == 'B'
+        assert htget(h, 1234) == lp1
+        htset(h, 1234, ffi.NULL, tl0)
+        self.commit_transaction()
+        #
+        self.start_transaction()
+        stm_major_collect()       # to get rid of the hashtable object
+
+    def test_conflict(self):
+        lp1 = stm_allocate_old(16)
+        lp2 = stm_allocate_old(16)
+        #
+        self.start_transaction()
+        h = self.allocate_hashtable()
+        self.push_root(h)
+        self.commit_transaction()
+        #
+        self.start_transaction()
+        h = self.pop_root()
+        self.push_root(h)
+        tl0 = self.tls[self.current_thread]
+        htset(h, 1234, lp1, tl0)
+        #
+        self.switch(1)
+        self.start_transaction()
+        tl1 = self.tls[self.current_thread]
+        htset(h, 1234, lp2, tl1)
+        #
+        self.switch(0)
+        self.commit_transaction()
+        #
+        py.test.raises(Conflict, self.switch, 1)
+        #
+        self.switch(0)
+        self.start_transaction()
+        self.pop_root()
+        stm_major_collect()       # to get rid of the hashtable object
+        self.commit_transaction()
+
+    def test_keepalive_minor(self):
+        self.start_transaction()
+        h = self.allocate_hashtable()
+        self.push_root(h)
+        lp1 = stm_allocate(16)
+        stm_set_char(lp1, 'N')
+        tl0 = self.tls[self.current_thread]
+        htset(h, 1234, lp1, tl0)
+        stm_minor_collect()
+        h = self.pop_root()
+        lp1b = htget(h, 1234)
+        assert lp1b != ffi.NULL
+        assert stm_get_char(lp1b) == 'N'
+        assert lp1b != lp1
+
+    def test_keepalive_major(self):
+        lp1 = stm_allocate_old(16)
+        #
+        self.start_transaction()
+        h = self.allocate_hashtable()
+        self.push_root(h)
+        stm_set_char(lp1, 'N')
+        tl0 = self.tls[self.current_thread]
+        htset(h, 1234, lp1, tl0)
+        self.commit_transaction()
+        #
+        self.start_transaction()
+        stm_major_collect()
+        h = self.pop_root()
+        lp1b = htget(h, 1234)
+        assert lp1b == lp1
+        assert stm_get_char(lp1b) == 'N'
+        #
+        stm_major_collect()       # to get rid of the hashtable object
+        self.commit_transaction()
+
+    def test_minor_collect_bug1(self):
+        self.start_transaction()
+        lp1 = stm_allocate(32)
+        self.push_root(lp1)
+        h = self.allocate_hashtable()
+        self.push_root(h)
+        stm_minor_collect()
+        h = self.pop_root()
+        lp1 = self.pop_root()
+        print 'h', h                       # 0xa040010
+        print 'lp1', lp1                   # 0xa040040
+        tl0 = self.tls[self.current_thread]
+        htset(h, 1, lp1, tl0)
+        self.commit_transaction()
+        #
+        self.start_transaction()
+        assert htget(h, 1) == lp1
+        stm_major_collect()       # to get rid of the hashtable object
+
+    def test_minor_collect_bug1_different_thread(self):
+        self.start_transaction()
+        lp1 = stm_allocate(32)
+        self.push_root(lp1)
+        h = self.allocate_hashtable()
+        self.push_root(h)
+        stm_minor_collect()
+        h = self.pop_root()
+        lp1 = self.pop_root()
+        print 'h', h                       # 0xa040010
+        print 'lp1', lp1                   # 0xa040040
+        tl0 = self.tls[self.current_thread]
+        htset(h, 1, lp1, tl0)
+        self.commit_transaction()
+        #
+        self.switch(1)            # in a different thread
+        self.start_transaction()
+        assert htget(h, 1) == lp1
+        stm_major_collect()       # to get rid of the hashtable object
+
+    def test_major_collect_bug2(self):
+        self.start_transaction()
+        lp1 = stm_allocate(24)
+        self.push_root(lp1)
+        self.commit_transaction()
+        lp1 = self.pop_root()
+        #
+        self.switch(1)
+        self.start_transaction()
+        stm_write(lp1)            # force this page to be shared
+        #
+        self.switch(0)
+        self.start_transaction()
+        h = self.allocate_hashtable()
+        tl0 = self.tls[self.current_thread]
+        htset(h, 10, stm_allocate(32), tl0)
+        htset(h, 11, stm_allocate(32), tl0)
+        htset(h, 12, stm_allocate(32), tl0)
+        self.push_root(h)
+        #
+        self.switch(1)            # in a different thread
+        stm_major_collect()       # force a _stm_rehash_hashtable()
+        #
+        self.switch(0)            # back to the original thread
+        h = self.pop_root()
+        assert htget(h, 10) != ffi.NULL
+        assert htget(h, 11) != ffi.NULL
+        assert htget(h, 12) != ffi.NULL
+
+    def test_list_1(self):
+        self.start_transaction()
+        h = self.allocate_hashtable()
+        tl0 = self.tls[self.current_thread]
+        for i in range(32):
+            assert ht_length_upper_bound(h) == i
+            htset(h, 19 ^ i, stm_allocate(32), tl0)
+        assert ht_length_upper_bound(h) == 32
+
+    def test_list_2(self):
+        self.start_transaction()
+        h = self.allocate_hashtable()
+        tl0 = self.tls[self.current_thread]
+        expected = []
+        for i in range(29):
+            lp = stm_allocate(32)
+            htset(h, 19 ^ i, lp, tl0)
+            expected.append((19 ^ i, lp))
+        lst = htitems(h)
+        assert len(lst) == 29
+        assert sorted(lst) == sorted(expected)
+
+    def test_list_3(self):
+        self.start_transaction()
+        h = self.allocate_hashtable()
+        tl0 = self.tls[self.current_thread]
+        for i in range(29):
+            htset(h, 19 ^ i, stm_allocate(32), tl0)
+        assert htlen(h) == 29
+
+    def test_len_conflicts_with_additions(self):
+        self.start_transaction()
+        h = self.allocate_hashtable()
+        self.push_root(h)
+        self.commit_transaction()
+        #
+        self.start_transaction()
+        h = self.pop_root()
+        self.push_root(h)
+        tl0 = self.tls[self.current_thread]
+        htset(h, 10, stm_allocate(32), tl0)
+        #
+        self.switch(1)
+        self.start_transaction()
+        assert htlen(h) == 0
+        #
+        self.switch(0)
+        self.commit_transaction()
+        #
+        py.test.raises(Conflict, self.switch, 1)
+        #
+        self.switch(0)
+        self.start_transaction()
+        self.pop_root()
+        stm_major_collect()       # to get rid of the hashtable object
+        self.commit_transaction()
+
+    def test_grow_without_conflict(self):
+        self.start_transaction()
+        h = self.allocate_hashtable()
+        self.push_root(h)
+        self.commit_transaction()
+        h = self.pop_root()
+        self.push_root(h)
+        #
+        STEPS = 50
+        for i in range(STEPS):
+            self.switch(1)
+            self.start_transaction()
+            tl0 = self.tls[self.current_thread]
+            htset(h, i + STEPS, stm_allocate(32), tl0)
+            #
+            self.switch(0)
+            self.start_transaction()
+            tl0 = self.tls[self.current_thread]
+            htset(h, i, stm_allocate(24), tl0)
+            #
+            self.switch(1)
+            self.commit_transaction()
+            #
+            self.switch(0)
+            self.commit_transaction()
+        #
+        self.pop_root()
+        self.start_transaction()
+        stm_major_collect()       # to get rid of the hashtable object
+
+
+class TestRandomHashtable(BaseTestHashtable):
+
+    def setup_method(self, meth):
+        BaseTestHashtable.setup_method(self, meth)
+        self.values = []
+        self.mirror = None
+        self.roots = []
+        self.other_thread = ([], [])
+
+    def push_roots(self):
+        assert self.roots is None
+        self.roots = []
+        for k, hitems in self.mirror.items():
+            assert lib._get_type_id(k) == 421419
+            for key, value in hitems.items():
+                assert lib._get_type_id(value) < 1000
+                self.push_root(value)
+                self.roots.append(key)
+            self.push_root(k)
+            self.roots.append(None)
+        for v in self.values:
+            self.push_root(v)
+        self.mirror = None
+
+    def pop_roots(self):
+        assert self.mirror is None
+        for i in reversed(range(len(self.values))):
+            self.values[i] = self.pop_root()
+            assert stm_get_char(self.values[i]) == chr((i + 1) & 255)
+        self.mirror = {}
+        for r in reversed(self.roots):
+            obj = self.pop_root()
+            if r is None:
+                assert lib._get_type_id(obj) == 421419
+                self.mirror[obj] = curhitems = {}
+            else:
+                assert lib._get_type_id(obj) < 1000
+                curhitems[r] = obj
+        self.roots = None
+
+    def exchange_threads(self):
+        old_thread = (self.values, self.roots)
+        self.switch(1 - self.current_thread)
+        (self.values, self.roots) = self.other_thread
+        self.mirror = None
+        self.other_thread = old_thread
+        print "----- switch to %s -----" % (self.current_thread,)
+
+    def test_random_single_thread(self):
+        import random
+        #
+        for i in range(100):
+            print "start_transaction"
+            self.start_transaction()
+            self.pop_roots()
+            for j in range(10):
+                r = random.random()
+                if r < 0.05:
+                    h = self.allocate_hashtable()
+                    print "allocate_hashtable ->", h
+                    self.mirror[h] = {}
+                elif r < 0.10:
+                    print "stm_minor_collect"
+                    self.push_roots()
+                    stm_minor_collect()
+                    self.pop_roots()
+                elif r < 0.11:
+                    print "stm_major_collect"
+                    self.push_roots()
+                    stm_major_collect()
+                    self.pop_roots()
+                elif r < 0.5:
+                    if not self.mirror: continue
+                    h = random.choice(self.mirror.keys())
+                    if not self.mirror[h]: continue
+                    key = random.choice(self.mirror[h].keys())
+                    value = self.mirror[h][key]
+                    print "htget(%r, %r) == %r" % (h, key, value)
+                    self.push_roots()
+                    self.push_root(value)
+                    result = htget(h, key)
+                    value = self.pop_root()
+                    assert result == value
+                    self.pop_roots()
+                elif r < 0.6:
+                    if not self.mirror: continue
+                    h = random.choice(self.mirror.keys())
+                    key = random.randrange(0, 40)
+                    if key in self.mirror[h]: continue
+                    print "htget(%r, %r) == NULL" % (h, key)
+                    self.push_roots()
+                    assert htget(h, key) == ffi.NULL
+                    self.pop_roots()
+                elif r < 0.63:
+                    if not self.mirror: continue
+                    h, _ = self.mirror.popitem()
+                    print "popped", h
+                elif r < 0.75:
+                    obj = stm_allocate(32)
+                    self.values.append(obj)
+                    stm_set_char(obj, chr(len(self.values) & 255))
+                else:
+                    if not self.mirror or not self.values: continue
+                    h = random.choice(self.mirror.keys())
+                    key = random.randrange(0, 32)
+                    value = random.choice(self.values)
+                    print "htset(%r, %r, %r)" % (h, key, value)
+                    self.push_roots()
+                    tl = self.tls[self.current_thread]
+                    htset(h, key, value, tl)
+                    self.pop_roots()
+                    self.mirror[h][key] = value
+            self.push_roots()
+            print "commit_transaction"
+            self.commit_transaction()
+        #
+        self.start_transaction()
+        self.become_inevitable()
+        self.pop_roots()
+        stm_major_collect()       # to get rid of the hashtable objects
+
+    def test_random_multiple_threads(self):
+        from random import randrange, Random
+        seed = randrange(0, 10000)
+        print "----------------------------------------- seed:", seed
+        random = Random(seed)
+        #
+        self.start_transaction()
+        self.exchange_threads()
+        self.start_transaction()
+        self.pop_roots()
+        #
+        for j in range(1000):
+            r = random.random()
+            if r > 0.9:
+                if r > 0.95:
+                    self.push_roots()
+                    self.commit_transaction()
+                    self.start_transaction()
+                    self.pop_roots()
+                else:
+                    self.push_roots()
+                    self.exchange_threads()
+                    self.pop_roots()
+                continue
+
+            if r < 0.05:
+                h = self.allocate_hashtable()
+                print "allocate_hashtable -> %r/%r" % (h, get_hashtable(h))
+                self.mirror[h] = {}
+            elif r < 0.10:
+                print "stm_minor_collect"
+                self.push_roots()
+                stm_minor_collect()
+                self.pop_roots()
+            elif r < 0.11:
+                print "stm_major_collect"
+                self.push_roots()
+                stm_major_collect()
+                self.pop_roots()
+            elif r < 0.5:
+                if not self.mirror: continue
+                h = random.choice(self.mirror.keys())
+                if not self.mirror[h]: continue
+                key = random.choice(self.mirror[h].keys())
+                value = self.mirror[h][key]
+                print "htget(%r/%r, %r) == %r" % (h, get_hashtable(h), key, value)
+                self.push_roots()
+                self.push_root(value)
+                result = htget(h, key)
+                value = self.pop_root()
+                assert result == value
+                self.pop_roots()
+            elif r < 0.6:
+                if not self.mirror: continue
+                h = random.choice(self.mirror.keys())
+                key = random.randrange(0, 40)
+                if key in self.mirror[h]: continue
+                print "htget(%r/%r, %r) == NULL" % (h, get_hashtable(h), key)
+                self.push_roots()
+                assert htget(h, key) == ffi.NULL
+                self.pop_roots()
+            elif r < 0.63:
+                if not self.mirror: continue
+                h, _ = self.mirror.popitem()
+                print "popped", h
+            elif r < 0.75:
+                obj = stm_allocate(32)
+                self.values.append(obj)
+                stm_set_char(obj, chr(len(self.values) & 255))
+            else:
+                if not self.mirror or not self.values: continue
+                h = random.choice(self.mirror.keys())
+                key = random.randrange(0, 32)
+                value = random.choice(self.values)
+                print "htset(%r/%r, %r, %r)" % (h, get_hashtable(h), key, value)
+                self.push_roots()
+                tl = self.tls[self.current_thread]
+                htset(h, key, value, tl)
+                self.pop_roots()
+                self.mirror[h][key] = value
+        #
+        print 'closing down...'
+        self.become_inevitable()
+        self.commit_transaction()
+        self.exchange_threads()
+        self.pop_roots()
+        self.become_inevitable()
+        stm_major_collect()       # to get rid of the hashtable objects

From noreply at buildbot.pypy.org  Thu Apr  9 16:32:13 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  9 Apr 2015 16:32:13 +0200 (CEST)
Subject: [pypy-commit] pypy default: Print more location info to stderr
Message-ID: <20150409143213.F2B3C1C1513@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76765:b8efc7b61f61
Date: 2015-04-09 16:32 +0200
http://bitbucket.org/pypy/pypy/changeset/b8efc7b61f61/

Log:	Print more location info to stderr

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -1,5 +1,6 @@
 """Python control flow graph generation and bytecode assembly."""
 
+import os
 from rpython.rlib import rfloat
 from rpython.rlib.objectmodel import we_are_translated
 
@@ -392,6 +393,8 @@
         for block in blocks:
             depth = self._do_stack_depth_walk(block)
             if block.auto_inserted_return and depth != 0:
+                os.write(2, "StackDepthComputationError in %s at %s:%s\n" % (
+                    self.compile_info.filename, self.name, self.first_lineno))
                 raise StackDepthComputationError   # fatal error
         return self._max_depth
 

From noreply at buildbot.pypy.org  Thu Apr  9 16:57:10 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  9 Apr 2015 16:57:10 +0200 (CEST)
Subject: [pypy-commit] pypy default: Next test and fix
Message-ID: <20150409145710.328121C116D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76766:bcdcaac84be7
Date: 2015-04-09 16:56 +0200
http://bitbucket.org/pypy/pypy/changeset/bcdcaac84be7/

Log:	Next test and fix

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -404,14 +404,13 @@
 
     def _do_stack_depth_walk(self, block):
         depth = block.initial_depth
-        done = False
         for instr in block.instructions:
             depth += _opcode_stack_effect(instr.opcode, instr.arg)
             if depth >= self._max_depth:
                 self._max_depth = depth
+            jump_op = instr.opcode
             if instr.has_jump:
                 target_depth = depth
-                jump_op = instr.opcode
                 if jump_op == ops.FOR_ITER:
                     target_depth -= 2
                 elif (jump_op == ops.SETUP_FINALLY or
@@ -428,10 +427,13 @@
                 self._next_stack_depth_walk(instr.jump[0], target_depth)
                 if jump_op == ops.JUMP_ABSOLUTE or jump_op == ops.JUMP_FORWARD:
                     # Nothing more can occur.
-                    done = True
                     break
-        if block.next_block and not done:
-            self._next_stack_depth_walk(block.next_block, depth)
+            elif jump_op == ops.RETURN_VALUE or jump_op == ops.RAISE_VARARGS:
+                # Nothing more can occur.
+                break
+        else:
+            if block.next_block:
+                self._next_stack_depth_walk(block.next_block, depth)
         return depth
 
     def _build_lnotab(self, blocks):
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -819,6 +819,13 @@
         code = compile_with_astcompiler(source, 'exec', self.space)
         assert code.co_stacksize == 1
 
+    def test_stackeffect_bug7(self):
+        source = '''def f():
+            for i in a:
+                return
+        '''
+        code = compile_with_astcompiler(source, 'exec', self.space)
+
     def test_lambda(self):
         yield self.st, "y = lambda x: x", "y(4)", 4
 

From noreply at buildbot.pypy.org  Thu Apr  9 17:00:46 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Thu,  9 Apr 2015 17:00:46 +0200 (CEST)
Subject: [pypy-commit] stmgc default: add some comments
Message-ID: <20150409150046.B279C1C116D@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: 
Changeset: r1754:e55658d12179
Date: 2015-04-09 17:03 +0200
http://bitbucket.org/pypy/stmgc/changeset/e55658d12179/

Log:	add some comments

diff --git a/c8/stm/hashtable.c b/c8/stm/hashtable.c
--- a/c8/stm/hashtable.c
+++ b/c8/stm/hashtable.c
@@ -109,6 +109,8 @@
 {
     /* can only be safely called during major GC, when all other threads
        are suspended */
+    assert(_has_mutex());
+
     long i;
     for (i = 1; i < NB_SEGMENTS; i++) {
         if (get_priv_segment(i)->transaction_state == TS_NONE)
@@ -174,6 +176,7 @@
         if (entry == NULL)
             continue;
         if (segment_base != NULL) {
+            /* -> compaction during major GC */
             if (((struct stm_hashtable_entry_s *)
                        REAL_ADDRESS(segment_base, entry))->object == NULL &&
                    !_stm_was_read_by_anybody((object_t *)entry)) {
@@ -283,6 +286,9 @@
         /* we can only enter here once!  If we allocate stuff, we may
            run the GC, and so 'hashtableobj' might move afterwards. */
         if (_is_in_nursery(hashtableobj)) {
+            /* this also means that the hashtable is from this
+               transaction and not visible to other segments yet, so
+               the new entry can be nursery-allocated. */
             entry = (stm_hashtable_entry_t *)
                 stm_allocate(sizeof(stm_hashtable_entry_t));
             entry->userdata = stm_hashtable_entry_userdata;

From noreply at buildbot.pypy.org  Thu Apr  9 17:12:28 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Thu,  9 Apr 2015 17:12:28 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: Merge with stmgc-c8-hashtable
Message-ID: <20150409151228.28CF21C0173@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: stmgc-c8
Changeset: r76767:0524ad7cc770
Date: 2015-04-09 17:10 +0200
http://bitbucket.org/pypy/pypy/changeset/0524ad7cc770/

Log:	Merge with stmgc-c8-hashtable

diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py
--- a/rpython/rlib/rstm.py
+++ b/rpython/rlib/rstm.py
@@ -259,14 +259,13 @@
                                       'freelist': _ll_hashtable_freelist,
                                         'lookup': _ll_hashtable_lookup,
                                       'writeobj': _ll_hashtable_writeobj})
-# NULL_HASHTABLE = lltype.nullptr(_HASHTABLE_OBJ)
-NULL_HASHTABLE = None
+NULL_HASHTABLE = lltype.nullptr(_HASHTABLE_OBJ)
 
 def _ll_hashtable_trace(gc, obj, callback, arg):
     from rpython.memory.gctransform.stmframework import get_visit_function
     visit_fn = get_visit_function(callback, arg)
     addr = obj + llmemory.offsetof(_HASHTABLE_OBJ, 'll_raw_hashtable')
-    llop.stm_hashtable_tracefn(lltype.Void, addr.address[0], visit_fn)
+    llop.stm_hashtable_tracefn(lltype.Void, obj, addr.address[0], visit_fn)
 lambda_hashtable_trace = lambda: _ll_hashtable_trace
 
 def _ll_hashtable_finalizer(h):
@@ -280,66 +279,22 @@
 def create_hashtable():
     if not we_are_translated():
         return HashtableForTest()      # for tests
-    return HashtableEmulation()
-    # rgc.register_custom_light_finalizer(_HASHTABLE_OBJ, lambda_hashtable_finlz)
-    # rgc.register_custom_trace_hook(_HASHTABLE_OBJ, lambda_hashtable_trace)
-    # # Pass a null pointer to _STM_HASHTABLE_ENTRY to stm_hashtable_create().
-    # # Make sure we see a malloc() of it, so that its typeid is correctly
-    # # initialized.  It can be done in a NonConstant(False) path so that
-    # # the C compiler will actually drop it.
-    # if _false:
-    #     p = lltype.malloc(_STM_HASHTABLE_ENTRY)
-    # else:
-    #     p = lltype.nullptr(_STM_HASHTABLE_ENTRY)
-    # h = lltype.malloc(_HASHTABLE_OBJ, zero=True)
-    # h.ll_raw_hashtable = llop.stm_hashtable_create(_STM_HASHTABLE_P, p)
-    # return h
+    rgc.register_custom_light_finalizer(_HASHTABLE_OBJ, lambda_hashtable_finlz)
+    rgc.register_custom_trace_hook(_HASHTABLE_OBJ, lambda_hashtable_trace)
+    # Pass a null pointer to _STM_HASHTABLE_ENTRY to stm_hashtable_create().
+    # Make sure we see a malloc() of it, so that its typeid is correctly
+    # initialized.  It can be done in a NonConstant(False) path so that
+    # the C compiler will actually drop it.
+    if _false:
+        p = lltype.malloc(_STM_HASHTABLE_ENTRY)
+    else:
+        p = lltype.nullptr(_STM_HASHTABLE_ENTRY)
+    h = lltype.malloc(_HASHTABLE_OBJ, zero=True)
+    h.ll_raw_hashtable = llop.stm_hashtable_create(_STM_HASHTABLE_P, p)
+    return h
 
 NULL_GCREF = lltype.nullptr(llmemory.GCREF.TO)
 
-class HashtableEmulation(object):
-    def __init__(self):
-        self._content = {}      # dict {integer: GCREF}
-
-    def get(self, key):
-        return self._content.get(key, NULL_GCREF)
-
-    def set(self, key, value):
-        if value:
-            self._content[key] = value
-        else:
-            try:
-                del self._content[key]
-            except KeyError:
-                pass
-
-    def len(self):
-        return len(self._content)
-
-    def list(self):
-        items = []
-        for key in self._content.keys():
-            items.append(self.lookup(key))
-        count = len(items)
-        return items, count
-
-    def freelist(self, array):
-        pass
-
-    def lookup(self, key):
-        return EntryObjectEmulation(self, key)
-
-    def writeobj(self, entry, nvalue):
-        self.set(entry.key, nvalue)
-
-class EntryObjectEmulation(object):
-    def __init__(self, hashtable, key):
-        self.hashtable = hashtable
-        self.key = key
-        self.index = r_uint(key)
-        self.object = hashtable.get(key)
-
-
 class HashtableForTest(object):
     def __init__(self):
         self._content = {}      # dict {integer: GCREF}
diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py
--- a/rpython/translator/stm/funcgen.py
+++ b/rpython/translator/stm/funcgen.py
@@ -343,8 +343,9 @@
     arg1 = funcgen.expr(op.args[1])
     arg2 = funcgen.expr(op.args[2])
     result = funcgen.expr(op.result)
-    return '%s = stm_hashtable_lookup((object_t *)%s, %s, %s);' % (
-        result, arg0, arg1, arg2)
+    typename = cdecl(funcgen.lltypename(op.result), '')
+    return '%s = (%s)stm_hashtable_lookup((object_t *)%s, %s, %s);' % (
+        result, typename, arg0, arg1, arg2)
 
 def stm_hashtable_length_upper_bound(funcgen, op):
     arg0 = funcgen.expr(op.args[0])
@@ -357,10 +358,12 @@
     arg1 = funcgen.expr(op.args[1])
     arg2 = funcgen.expr(op.args[2])
     result = funcgen.expr(op.result)
-    return '%s = stm_hashtable_list((object_t *)%s, %s, %s);' % (
-        result, arg0, arg1, arg2)
+    return ('%s = stm_hashtable_list((object_t *)%s, %s, '
+            '(stm_hashtable_entry_t **)%s);' % (result, arg0, arg1, arg2))
 
 def stm_hashtable_tracefn(funcgen, op):
     arg0 = funcgen.expr(op.args[0])
     arg1 = funcgen.expr(op.args[1])
-    return 'stm_hashtable_tracefn((stm_hashtable_t *)%s, %s);' % (arg0, arg1)
+    arg2 = funcgen.expr(op.args[2])
+    return ('stm_hashtable_tracefn(%s, (stm_hashtable_t *)%s, '
+            ' (void(*)(object_t**))%s);' % (arg0, arg1, arg2))
diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c
--- a/rpython/translator/stm/src_stm/stm/core.c
+++ b/rpython/translator/stm/src_stm/stm/core.c
@@ -280,8 +280,14 @@
         struct stm_undo_s *end = undo + cl->written_count;
         for (; undo < end; undo++) {
             if (undo->type == TYPE_POSITION_MARKER) {
-                fprintf(stderr, "    marker %p %lu\n",
-                        undo->marker_object, undo->marker_odd_number);
+                if (undo->type2 == TYPE_MODIFIED_HASHTABLE) {
+                    fprintf(stderr, "    hashtable %p\n",
+                            undo->modif_hashtable);
+                }
+                else {
+                    fprintf(stderr, "    marker %p %lu\n",
+                            undo->marker_object, undo->marker_odd_number);
+                }
                 continue;
             }
             fprintf(stderr, "    obj %p, size %d, ofs %lu: ", undo->object,
@@ -383,21 +389,40 @@
                     struct stm_undo_s *undo = cl->written;
                     struct stm_undo_s *end = cl->written + cl->written_count;
                     for (; undo < end; undo++) {
-                        if (undo->type == TYPE_POSITION_MARKER)
+                        object_t *obj;
+
+                        if (undo->type != TYPE_POSITION_MARKER) {
+                            /* common case: 'undo->object' was written to
+                               in this past commit, so we must check that
+                               it was not read by us. */
+                            obj = undo->object;
+                        }
+                        else if (undo->type2 != TYPE_MODIFIED_HASHTABLE)
                             continue;
-                        if (_stm_was_read(undo->object)) {
-                            /* first reset all modified objects from the backup
-                               copies as soon as the first conflict is detected;
-                               then we will proceed below to update our segment from
-                               the old (but unmodified) version to the newer version.
-                            */
-                            reset_modified_from_backup_copies(my_segnum);
-                            timing_write_read_contention(cl->written, undo);
-                            needs_abort = true;
+                        else {
+                            /* the previous stm_undo_s is about a written
+                               'entry' object, which belongs to the hashtable
+                               given now.  Check that we haven't read the
+                               hashtable (via stm_hashtable_list()). */
+                            obj = undo->modif_hashtable;
+                        }
 
-                            dprintf(("_stm_validate() failed for obj %p\n", undo->object));
-                            break;
-                        }
+                        if (LIKELY(!_stm_was_read(obj)))
+                            continue;
+
+                        /* conflict! */
+                        dprintf(("_stm_validate() failed for obj %p\n", obj));
+
+                        /* first reset all modified objects from the backup
+                           copies as soon as the first conflict is detected;
+                           then we will proceed below to update our segment
+                           from the old (but unmodified) version to the newer
+                           version.
+                        */
+                        reset_modified_from_backup_copies(my_segnum);
+                        timing_write_read_contention(cl->written, undo);
+                        needs_abort = true;
+                        break;
                     }
                 }
 
diff --git a/rpython/translator/stm/src_stm/stm/core.h b/rpython/translator/stm/src_stm/stm/core.h
--- a/rpython/translator/stm/src_stm/stm/core.h
+++ b/rpython/translator/stm/src_stm/stm/core.h
@@ -190,9 +190,15 @@
         uintptr_t marker_odd_number; /* the odd number part of the marker */
         object_t *marker_object;     /* the object part of the marker */
     };
+    struct {
+        intptr_t type1;             /* TYPE_POSITION_MARKER (again) */
+        intptr_t type2;             /* TYPE_MODIFIED_HASHTABLE */
+        object_t *modif_hashtable;  /* modified entry is previous stm_undo_s */
+    };
   };
 };
 #define TYPE_POSITION_MARKER    (-1)
+#define TYPE_MODIFIED_HASHTABLE (-2)
 #define SLICE_OFFSET(slice)  ((slice) >> 16)
 #define SLICE_SIZE(slice)    ((int)((slice) & 0xFFFF))
 #define NEW_SLICE(offset, size) (((uint64_t)(offset)) << 16 | (size))
@@ -251,6 +257,14 @@
     return stm_object_pages + segment_num * (NB_PAGES * 4096UL);
 }
 
+static inline long get_num_segment_containing_address(char *addr)
+{
+    uintptr_t delta = addr - stm_object_pages;
+    uintptr_t result = delta / (NB_PAGES * 4096UL);
+    assert(result < NB_SEGMENTS);
+    return result;
+}
+
 static inline
 struct stm_segment_info_s *get_segment(long segment_num) {
     return (struct stm_segment_info_s *)REAL_ADDRESS(
@@ -285,6 +299,17 @@
 static void _signal_handler(int sig, siginfo_t *siginfo, void *context);
 static bool _stm_validate();
 
+static inline bool was_read_remote(char *base, object_t *obj)
+{
+    uint8_t other_transaction_read_version =
+        ((struct stm_segment_info_s *)REAL_ADDRESS(base, STM_PSEGMENT))
+            ->transaction_read_version;
+    uint8_t rm = ((struct stm_read_marker_s *)
+                  (base + (((uintptr_t)obj) >> 4)))->rm;
+    assert(rm <= other_transaction_read_version);
+    return rm == other_transaction_read_version;
+}
+
 static inline void _duck(void) {
     /* put a call to _duck() between two instructions that set 0 into
        a %gs-prefixed address and that may otherwise be replaced with
diff --git a/rpython/translator/stm/src_stm/stm/gcpage.c b/rpython/translator/stm/src_stm/stm/gcpage.c
--- a/rpython/translator/stm/src_stm/stm/gcpage.c
+++ b/rpython/translator/stm/src_stm/stm/gcpage.c
@@ -127,6 +127,58 @@
     return o;
 }
 
+static void _fill_preexisting_slice(long segnum, char *dest,
+                                    const char *src, uintptr_t size)
+{
+    uintptr_t np = dest - get_segment_base(segnum);
+    if (get_page_status_in(segnum, np / 4096) != PAGE_NO_ACCESS)
+        memcpy(dest, src, size);
+}
+
+object_t *stm_allocate_preexisting(ssize_t size_rounded_up,
+                                   const char *initial_data)
+{
+    stm_char *np = allocate_outside_nursery_large(size_rounded_up);
+    uintptr_t nobj = (uintptr_t)np;
+    dprintf(("allocate_preexisting: %p\n", (object_t *)nobj));
+
+    char *nobj_seg0 = stm_object_pages + nobj;
+    memcpy(nobj_seg0, initial_data, size_rounded_up);
+    ((struct object_s *)nobj_seg0)->stm_flags = GCFLAG_WRITE_BARRIER;
+
+    acquire_privatization_lock(STM_SEGMENT->segment_num);
+    DEBUG_EXPECT_SEGFAULT(false);
+
+    long j;
+    for (j = 1; j < NB_SEGMENTS; j++) {
+        const char *src = nobj_seg0;
+        char *dest = get_segment_base(j) + nobj;
+        char *end = dest + size_rounded_up;
+
+        while (((uintptr_t)dest) / 4096 != ((uintptr_t)end - 1) / 4096) {
+            uintptr_t count = 4096 - (((uintptr_t)dest) & 4095);
+            _fill_preexisting_slice(j, dest, src, count);
+            src += count;
+            dest += count;
+        }
+        _fill_preexisting_slice(j, dest, src, end - dest);
+
+#ifdef STM_TESTS
+        /* can't really enable this check outside tests, because there is
+           a change that the transaction_state changes in parallel */
+        if (get_priv_segment(j)->transaction_state != TS_NONE) {
+            assert(!was_read_remote(get_segment_base(j), (object_t *)nobj));
+        }
+#endif
+    }
+
+    DEBUG_EXPECT_SEGFAULT(true);
+    release_privatization_lock(STM_SEGMENT->segment_num);
+
+    write_fence();     /* make sure 'nobj' is fully initialized from
+                          all threads here */
+    return (object_t *)nobj;
+}
 
 /************************************************************/
 
@@ -246,6 +298,8 @@
 }
 
 
+#define TRACE_FOR_MAJOR_COLLECTION  (&mark_record_trace)
+
 static void mark_and_trace(
     object_t *obj,
     char *segment_base, /* to trace obj in */
@@ -405,7 +459,8 @@
         struct stm_undo_s *modified = (struct stm_undo_s *)lst->items;
         struct stm_undo_s *end = (struct stm_undo_s *)(lst->items + lst->count);
         for (; modified < end; modified++) {
-            if (modified->type == TYPE_POSITION_MARKER)
+            if (modified->type == TYPE_POSITION_MARKER &&
+                    modified->type2 != TYPE_MODIFIED_HASHTABLE)
                 mark_visit_possibly_new_object(modified->marker_object, pseg);
         }
     }
@@ -538,6 +593,31 @@
                 list_set_item(lst, n, list_pop_item(lst));
             }
         }
+
+        /* Remove from 'modified_old_objects' all old hashtables that die */
+        {
+            lst = pseg->modified_old_objects;
+            uintptr_t j, k = 0, limit = list_count(lst);
+            for (j = 0; j < limit; j += 3) {
+                uintptr_t e0 = list_item(lst, j + 0);
+                uintptr_t e1 = list_item(lst, j + 1);
+                uintptr_t e2 = list_item(lst, j + 2);
+                if (e0 == TYPE_POSITION_MARKER &&
+                    e1 == TYPE_MODIFIED_HASHTABLE &&
+                    !mark_visited_test((object_t *)e2)) {
+                    /* hashtable object dies */
+                }
+                else {
+                    if (j != k) {
+                        list_set_item(lst, k + 0, e0);
+                        list_set_item(lst, k + 1, e1);
+                        list_set_item(lst, k + 2, e2);
+                    }
+                    k += 3;
+                }
+            }
+            lst->count = k;
+        }
     }
 #pragma pop_macro("STM_SEGMENT")
 #pragma pop_macro("STM_PSEGMENT")
diff --git a/rpython/translator/stm/src_stm/stm/hashtable.c b/rpython/translator/stm/src_stm/stm/hashtable.c
new file mode 100644
--- /dev/null
+++ b/rpython/translator/stm/src_stm/stm/hashtable.c
@@ -0,0 +1,532 @@
+/* Imported by rpython/translator/stm/import_stmgc.py */
+/*
+Design of stmgc's "hashtable" objects
+=====================================
+A "hashtable" is theoretically a lazily-filled array of objects of
+length 2**64.  Initially it is full of NULLs.  It's obviously
+implemented as a dictionary in which NULL objects are not needed.
+
+A real dictionary can be implemented on top of it, by using the index
+`hash(key)` in the hashtable, and storing a list of `(key, value)`
+pairs at that index (usually only one, unless there is a hash
+collision).
+
+The main operations on a hashtable are reading or writing an object at a
+given index.  It also supports fetching the list of non-NULL entries.
+
+There are two markers for every index (a read and a write marker).
+This is unlike regular arrays, which have only two markers in total.
+
+Additionally, we use the read marker for the hashtable object itself
+to mean "we have read the complete list of keys".  This plays the role
+of a "global" read marker: when any thread adds a new key/value object
+to the hashtable, this new object's read marker is initialized with a
+copy of the "global" read marker --- in all segments.
+
+
+Implementation
+--------------
+
+First idea: have the hashtable in raw memory, pointing to "entry"
+objects (which are regular, GC- and STM-managed objects).  The entry
+objects themselves point to the user-specified objects.  The entry
+objects hold the read/write markers.  Every entry object, once
+created, stays around.  It is only removed by the next major GC if it
+points to NULL and its read/write markers are not set in any
+currently-running transaction.
+
+References
+----------
+
+Inspired by: http://ppl.stanford.edu/papers/podc011-bronson.pdf
+*/
+
+
+uint32_t stm_hashtable_entry_userdata;
+
+
+#define INITIAL_HASHTABLE_SIZE   8
+#define PERTURB_SHIFT            5
+#define RESIZING_LOCK            0
+
+typedef struct {
+    uintptr_t mask;
+
+    /* 'resize_counter' start at an odd value, and is decremented (by
+       6) for every new item put in 'items'.  When it crosses 0, we
+       instead allocate a bigger table and change 'resize_counter' to
+       be a regular pointer to it (which is then even).  The whole
+       structure is immutable then.
+
+       The field 'resize_counter' also works as a write lock: changes
+       go via the intermediate value RESIZING_LOCK (0).
+    */
+    uintptr_t resize_counter;
+
+    stm_hashtable_entry_t *items[INITIAL_HASHTABLE_SIZE];
+} stm_hashtable_table_t;
+
+#define IS_EVEN(p) (((p) & 1) == 0)
+
+struct stm_hashtable_s {
+    stm_hashtable_table_t *table;
+    stm_hashtable_table_t initial_table;
+    uint64_t additions;
+};
+
+
+static inline void init_table(stm_hashtable_table_t *table, uintptr_t itemcount)
+{
+    table->mask = itemcount - 1;
+    table->resize_counter = itemcount * 4 + 1;
+    memset(table->items, 0, itemcount * sizeof(stm_hashtable_entry_t *));
+}
+
+stm_hashtable_t *stm_hashtable_create(void)
+{
+    stm_hashtable_t *hashtable = malloc(sizeof(stm_hashtable_t));
+    assert(hashtable);
+    hashtable->table = &hashtable->initial_table;
+    hashtable->additions = 0;
+    init_table(&hashtable->initial_table, INITIAL_HASHTABLE_SIZE);
+    return hashtable;
+}
+
+void stm_hashtable_free(stm_hashtable_t *hashtable)
+{
+    uintptr_t rc = hashtable->initial_table.resize_counter;
+    free(hashtable);
+    while (IS_EVEN(rc)) {
+        assert(rc != RESIZING_LOCK);
+
+        stm_hashtable_table_t *table = (stm_hashtable_table_t *)rc;
+        rc = table->resize_counter;
+        free(table);
+    }
+}
+
+static bool _stm_was_read_by_anybody(object_t *obj)
+{
+    /* can only be safely called during major GC, when all other threads
+       are suspended */
+    long i;
+    for (i = 1; i < NB_SEGMENTS; i++) {
+        if (get_priv_segment(i)->transaction_state == TS_NONE)
+            continue;
+        if (was_read_remote(get_segment_base(i), obj))
+            return true;
+    }
+    return false;
+}
+
+#define VOLATILE_HASHTABLE(p)    ((volatile stm_hashtable_t *)(p))
+#define VOLATILE_TABLE(p)  ((volatile stm_hashtable_table_t *)(p))
+
+static void _insert_clean(stm_hashtable_table_t *table,
+                          stm_hashtable_entry_t *entry,
+                          uintptr_t index)
+{
+    uintptr_t mask = table->mask;
+    uintptr_t i = index & mask;
+    if (table->items[i] == NULL) {
+        table->items[i] = entry;
+        return;
+    }
+
+    uintptr_t perturb = index;
+    while (1) {
+        i = (i << 2) + i + perturb + 1;
+        i &= mask;
+        if (table->items[i] == NULL) {
+            table->items[i] = entry;
+            return;
+        }
+
+        perturb >>= PERTURB_SHIFT;
+    }
+}
+
+static void _stm_rehash_hashtable(stm_hashtable_t *hashtable,
+                                  uintptr_t biggercount,
+                                  char *segment_base)
+{
+    dprintf(("rehash %p to size %ld, segment_base=%p\n",
+             hashtable, biggercount, segment_base));
+
+    size_t size = (offsetof(stm_hashtable_table_t, items)
+                   + biggercount * sizeof(stm_hashtable_entry_t *));
+    stm_hashtable_table_t *biggertable = malloc(size);
+    assert(biggertable);   // XXX
+
+    stm_hashtable_table_t *table = hashtable->table;
+    table->resize_counter = (uintptr_t)biggertable;
+    /* ^^^ this unlocks the table by writing a non-zero value to
+       table->resize_counter, but the new value is a pointer to the
+       new bigger table, so IS_EVEN() is still true */
+    assert(IS_EVEN(table->resize_counter));
+
+    init_table(biggertable, biggercount);
+
+    uintptr_t j, mask = table->mask;
+    uintptr_t rc = biggertable->resize_counter;
+    for (j = 0; j <= mask; j++) {
+        stm_hashtable_entry_t *entry = table->items[j];
+        if (entry == NULL)
+            continue;
+        if (segment_base != NULL) {
+            if (((struct stm_hashtable_entry_s *)
+                       REAL_ADDRESS(segment_base, entry))->object == NULL &&
+                   !_stm_was_read_by_anybody((object_t *)entry)) {
+                dprintf(("  removing dead %p\n", entry));
+                continue;
+            }
+        }
+
+        uintptr_t eindex;
+        if (segment_base == NULL)
+            eindex = entry->index;   /* read from STM_SEGMENT */
+        else
+            eindex = ((struct stm_hashtable_entry_s *)
+                       REAL_ADDRESS(segment_base, entry))->index;
+
+        dprintf(("  insert_clean %p at index=%ld\n",
+                 entry, eindex));
+        _insert_clean(biggertable, entry, eindex);
+        assert(rc > 6);
+        rc -= 6;
+    }
+    biggertable->resize_counter = rc;
+
+    write_fence();   /* make sure that 'biggertable' is valid here,
+                        and make sure 'table->resize_counter' is updated
+                        ('table' must be immutable from now on). */
+    VOLATILE_HASHTABLE(hashtable)->table = biggertable;
+}
+
+stm_hashtable_entry_t *stm_hashtable_lookup(object_t *hashtableobj,
+                                            stm_hashtable_t *hashtable,
+                                            uintptr_t index)
+{
+    stm_hashtable_table_t *table;
+    uintptr_t mask;
+    uintptr_t i;
+    stm_hashtable_entry_t *entry;
+
+ restart:
+    /* classical dict lookup logic */
+    table = VOLATILE_HASHTABLE(hashtable)->table;
+    mask = table->mask;      /* read-only field */
+    i = index & mask;
+    entry = VOLATILE_TABLE(table)->items[i];
+    if (entry != NULL) {
+        if (entry->index == index)
+            return entry;           /* found at the first try */
+
+        uintptr_t perturb = index;
+        while (1) {
+            i = (i << 2) + i + perturb + 1;
+            i &= mask;
+            entry = VOLATILE_TABLE(table)->items[i];
+            if (entry != NULL) {
+                if (entry->index == index)
+                    return entry;    /* found */
+            }
+            else
+                break;
+            perturb >>= PERTURB_SHIFT;
+        }
+    }
+    /* here, we didn't find the 'entry' with the correct index.  Note
+       that even if the same 'table' is modified or resized by other
+       threads concurrently, any new item found from a race condition
+       would anyway contain NULL in the present segment (ensured by
+       the first write_fence() below).  If the 'table' grows an entry
+       just after we checked above, then we go ahead and lock the
+       table; but after we get the lock, we will notice the new entry
+       (ensured by the second write_fence() below) and restart the
+       whole process.
+     */
+
+    uintptr_t rc = VOLATILE_TABLE(table)->resize_counter;
+
+    /* if rc is RESIZING_LOCK (which is 0, so even), a concurrent thread
+       is writing to the hashtable.  Or, if rc is another even number, it is
+       actually a pointer to the next version of the table, installed
+       just now.  In both cases, this thread must simply spin loop.
+    */
+    if (IS_EVEN(rc)) {
+        spin_loop();
+        goto restart;
+    }
+    /* in the other cases, we need to grab the RESIZING_LOCK.
+     */
+    if (!__sync_bool_compare_and_swap(&table->resize_counter,
+                                      rc, RESIZING_LOCK)) {
+        goto restart;
+    }
+    /* we now have the lock.  The only table with a non-even value of
+       'resize_counter' should be the last one in the chain, so if we
+       succeeded in locking it, check this. */
+    assert(table == hashtable->table);
+
+    /* Check that 'table->items[i]' is still NULL,
+       i.e. hasn't been populated under our feet.
+    */
+    if (table->items[i] != NULL) {
+        table->resize_counter = rc;    /* unlock */
+        goto restart;
+    }
+    /* if rc is greater than 6, there is enough room for a new
+       item in the current table.
+    */
+    if (rc > 6) {
+        /* we can only enter here once!  If we allocate stuff, we may
+           run the GC, and so 'hashtableobj' might move afterwards. */
+        if (_is_in_nursery(hashtableobj)) {
+            entry = (stm_hashtable_entry_t *)
+                stm_allocate(sizeof(stm_hashtable_entry_t));
+            entry->userdata = stm_hashtable_entry_userdata;
+            entry->index = index;
+            entry->object = NULL;
+        }
+        else {
+            /* for a non-nursery 'hashtableobj', we pretend that the
+               'entry' object we're about to return was already
+               existing all along, with NULL in all segments.  If the
+               caller of this function is going to modify the 'object'
+               field, it will call stm_write(entry) first, which will
+               correctly schedule 'entry' for write propagation.  We
+               do that even if 'hashtableobj' was created by the
+               running transaction: the new 'entry' object is created
+               as if it was older than the transaction.
+
+               Note the following difference: if 'hashtableobj' is
+               still in the nursery (case above), the 'entry' object
+               is also allocated from the nursery, and after a minor
+               collection it ages as an old-but-created-by-the-
+               current-transaction object.  We could try to emulate
+               this here, or to create young 'entry' objects, but
+               doing either of these would require careful
+               synchronization with other pieces of the code that may
+               change.
+            */
+            struct stm_hashtable_entry_s initial = {
+                .userdata = stm_hashtable_entry_userdata,
+                .index = index,
+                .object = NULL
+            };
+            entry = (stm_hashtable_entry_t *)
+                stm_allocate_preexisting(sizeof(stm_hashtable_entry_t),
+                                         (char *)&initial.header);
+            hashtable->additions++;
+        }
+        table->items[i] = entry;
+        write_fence();     /* make sure 'table->items' is written here */
+        VOLATILE_TABLE(table)->resize_counter = rc - 6;    /* unlock */
+        return entry;
+    }
+    else {
+        /* if rc is smaller than 6, we must allocate a new bigger table.
+         */
+        uintptr_t biggercount = table->mask + 1;
+        if (biggercount < 50000)
+            biggercount *= 4;
+        else
+            biggercount *= 2;
+        _stm_rehash_hashtable(hashtable, biggercount, /*segment_base=*/NULL);
+        goto restart;
+    }
+}
+
+object_t *stm_hashtable_read(object_t *hobj, stm_hashtable_t *hashtable,
+                             uintptr_t key)
+{
+    stm_hashtable_entry_t *e = stm_hashtable_lookup(hobj, hashtable, key);
+    stm_read((object_t *)e);
+    return e->object;
+}
+
+void stm_hashtable_write_entry(object_t *hobj, stm_hashtable_entry_t *entry,
+                               object_t *nvalue)
+{
+    if (_STM_WRITE_CHECK_SLOWPATH((object_t *)entry)) {
+
+        stm_write((object_t *)entry);
+
+        uintptr_t i = list_count(STM_PSEGMENT->modified_old_objects);
+        if (i > 0 && list_item(STM_PSEGMENT->modified_old_objects, i - 3)
+                     == (uintptr_t)entry) {
+            /* The stm_write() above recorded a write to 'entry'.  Here,
+               we add another stm_undo_s to modified_old_objects with
+               TYPE_MODIFIED_HASHTABLE.  It is ignored everywhere except
+               in _stm_validate().
+
+               The goal is that this TYPE_MODIFIED_HASHTABLE ends up in
+               the commit log's 'cl_written' array.  Later, another
+               transaction validating that log will check two things:
+
+               - the regular stm_undo_s entry put by stm_write() above
+                 will make the other transaction check that it didn't
+                 read the same 'entry' object;
+
+                 - the TYPE_MODIFIED_HASHTABLE entry we're adding now
+                   will make the other transaction check that it didn't
+                   do any stm_hashtable_list() on the complete hashtable.
+            */
+            STM_PSEGMENT->modified_old_objects = list_append3(
+                STM_PSEGMENT->modified_old_objects,
+                TYPE_POSITION_MARKER,      /* type1 */
+                TYPE_MODIFIED_HASHTABLE,   /* type2 */
+                (uintptr_t)hobj);          /* modif_hashtable */
+        }
+    }
+    entry->object = nvalue;
+}
+
+void stm_hashtable_write(object_t *hobj, stm_hashtable_t *hashtable,
+                         uintptr_t key, object_t *nvalue,
+                         stm_thread_local_t *tl)
+{
+    STM_PUSH_ROOT(*tl, nvalue);
+    STM_PUSH_ROOT(*tl, hobj);
+    stm_hashtable_entry_t *e = stm_hashtable_lookup(hobj, hashtable, key);
+    STM_POP_ROOT(*tl, hobj);
+    STM_POP_ROOT(*tl, nvalue);
+    stm_hashtable_write_entry(hobj, e, nvalue);
+}
+
+long stm_hashtable_length_upper_bound(stm_hashtable_t *hashtable)
+{
+    stm_hashtable_table_t *table;
+    uintptr_t rc;
+
+ restart:
+    table = VOLATILE_HASHTABLE(hashtable)->table;
+    rc = VOLATILE_TABLE(table)->resize_counter;
+    if (IS_EVEN(rc)) {
+        spin_loop();
+        goto restart;
+    }
+
+    uintptr_t initial_rc = (table->mask + 1) * 4 + 1;
+    uintptr_t num_entries_times_6 = initial_rc - rc;
+    return num_entries_times_6 / 6;
+}
+
+long stm_hashtable_list(object_t *hobj, stm_hashtable_t *hashtable,
+                        stm_hashtable_entry_t **results)
+{
+    /* Set the read marker.  It will be left as long as we're running
+       the same transaction.
+    */
+    stm_read(hobj);
+
+    /* Get the table.  No synchronization is needed: we may miss some
+       entries that are being added, but they would contain NULL in
+       this segment anyway. */
+    stm_hashtable_table_t *table = VOLATILE_HASHTABLE(hashtable)->table;
+
+    /* Read all entries, check which ones are not NULL, count them,
+       and optionally list them in 'results'.
+    */
+    uintptr_t i, mask = table->mask;
+    stm_hashtable_entry_t *entry;
+    long nresult = 0;
+
+    if (results != NULL) {
+        /* collect the results in the provided list */
+        for (i = 0; i <= mask; i++) {
+            entry = VOLATILE_TABLE(table)->items[i];
+            if (entry != NULL) {
+                stm_read((object_t *)entry);
+                if (entry->object != NULL)
+                    results[nresult++] = entry;
+            }
+        }
+    }
+    else {
+        /* don't collect, just get the exact number of results */
+        for (i = 0; i <= mask; i++) {
+            entry = VOLATILE_TABLE(table)->items[i];
+            if (entry != NULL) {
+                stm_read((object_t *)entry);
+                if (entry->object != NULL)
+                    nresult++;
+            }
+        }
+    }
+    return nresult;
+}
+
+static void _stm_compact_hashtable(struct object_s *hobj,
+                                   stm_hashtable_t *hashtable)
+{
+    stm_hashtable_table_t *table = hashtable->table;
+    uintptr_t rc = table->resize_counter;
+    assert(!IS_EVEN(rc));
+
+    if (hashtable->additions * 4 > table->mask) {
+        hashtable->additions = 0;
+
+        /* If 'hobj' was created in some current transaction, i.e. if it is
+           now an overflow object, then we have the risk that some of its
+           entry objects were not created with stm_allocate_preexisting().
+           In that situation, a valid workaround is to read all entry
+           objects in the segment of the running transaction.  Otherwise,
+           the base case is to read them all from segment zero.
+        */
+        long segnum = get_num_segment_containing_address((char *)hobj);
+        if (!IS_OVERFLOW_OBJ(get_priv_segment(segnum), hobj))
+            segnum = 0;
+
+        uintptr_t initial_rc = (table->mask + 1) * 4 + 1;
+        uintptr_t num_entries_times_6 = initial_rc - rc;
+        uintptr_t count = INITIAL_HASHTABLE_SIZE;
+        while (count * 4 < num_entries_times_6)
+            count *= 2;
+        /* sanity-check: 'num_entries_times_6 < initial_rc', and so 'count'
+           can never grow larger than the current table size. */
+        assert(count <= table->mask + 1);
+
+        dprintf(("compact with %ld items:\n", num_entries_times_6 / 6));
+        _stm_rehash_hashtable(hashtable, count, get_segment_base(segnum));
+    }
+
+    table = hashtable->table;
+    assert(!IS_EVEN(table->resize_counter));
+
+    if (table != &hashtable->initial_table) {
+        uintptr_t rc = hashtable->initial_table.resize_counter;
+        while (1) {
+            assert(IS_EVEN(rc));
+            assert(rc != RESIZING_LOCK);
+
+            stm_hashtable_table_t *old_table = (stm_hashtable_table_t *)rc;
+            if (old_table == table)
+                break;
+            rc = old_table->resize_counter;
+            free(old_table);
+        }
+        hashtable->initial_table.resize_counter = (uintptr_t)table;
+        assert(IS_EVEN(hashtable->initial_table.resize_counter));
+    }
+}
+
+void stm_hashtable_tracefn(struct object_s *hobj, stm_hashtable_t *hashtable,
+                           void trace(object_t **))
+{
+    if (trace == TRACE_FOR_MAJOR_COLLECTION)
+        _stm_compact_hashtable(hobj, hashtable);
+
+    stm_hashtable_table_t *table;
+    table = VOLATILE_HASHTABLE(hashtable)->table;
+
+    uintptr_t j, mask = table->mask;
+    for (j = 0; j <= mask; j++) {
+        stm_hashtable_entry_t *volatile *pentry;
+        pentry = &VOLATILE_TABLE(table)->items[j];
+        if (*pentry != NULL) {
+            trace((object_t **)pentry);
+        }
+    }
+}
diff --git a/rpython/translator/stm/src_stm/stm/marker.c b/rpython/translator/stm/src_stm/stm/marker.c
--- a/rpython/translator/stm/src_stm/stm/marker.c
+++ b/rpython/translator/stm/src_stm/stm/marker.c
@@ -42,7 +42,8 @@
     */
     while (contention != start) {
         --contention;
-        if (contention->type == TYPE_POSITION_MARKER) {
+        if (contention->type == TYPE_POSITION_MARKER &&
+            contention->type2 != TYPE_MODIFIED_HASHTABLE) {
             out_marker->odd_number = contention->marker_odd_number;
             out_marker->object = contention->marker_object;
             return;
@@ -69,6 +70,9 @@
             return;    /* already up-to-date */
     }
 
+    /* -2 is not odd */
+    assert(marker.odd_number != (uintptr_t)TYPE_MODIFIED_HASHTABLE);
+
     STM_PSEGMENT->position_markers_last = list_count(list);
     STM_PSEGMENT->modified_old_objects = list_append3(
         list,
diff --git a/rpython/translator/stm/src_stm/stm/misc.c b/rpython/translator/stm/src_stm/stm/misc.c
--- a/rpython/translator/stm/src_stm/stm/misc.c
+++ b/rpython/translator/stm/src_stm/stm/misc.c
@@ -31,10 +31,7 @@
 
 bool _stm_was_read(object_t *obj)
 {
-    uint8_t rm = ((struct stm_read_marker_s *)
-                  (STM_SEGMENT->segment_base + (((uintptr_t)obj) >> 4)))->rm;
-    assert(rm <= STM_SEGMENT->transaction_read_version);
-    return rm == STM_SEGMENT->transaction_read_version;
+    return was_read_remote(STM_SEGMENT->segment_base, obj);
 }
 
 bool _stm_was_written(object_t *obj)
diff --git a/rpython/translator/stm/src_stm/stm/nursery.c b/rpython/translator/stm/src_stm/stm/nursery.c
--- a/rpython/translator/stm/src_stm/stm/nursery.c
+++ b/rpython/translator/stm/src_stm/stm/nursery.c
@@ -424,6 +424,7 @@
     struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count);
 
     for (; undo < end; undo++) {
+        /* this logic also works if type2 == TYPE_MODIFIED_HASHTABLE */
         if (undo->type == TYPE_POSITION_MARKER)
             minor_trace_if_young(&undo->marker_object);
     }
diff --git a/rpython/translator/stm/src_stm/stm/pages.h b/rpython/translator/stm/src_stm/stm/pages.h
--- a/rpython/translator/stm/src_stm/stm/pages.h
+++ b/rpython/translator/stm/src_stm/stm/pages.h
@@ -62,7 +62,11 @@
 
 static inline bool get_page_status_in(long segnum, uintptr_t pagenum)
 {
-    /* reading page status requires "read"-lock: */
+    /* reading page status requires "read"-lock, which is defined as
+       "any segment has the privatization_lock".  This is enough to
+       prevent the "write"-lock from being acquired by somebody else
+       (defined as "_all_ segments have the privatization_lock").
+    */
     assert(STM_PSEGMENT->privatization_lock);
 
     OPT_ASSERT(segnum < 8 * sizeof(struct page_shared_s));
diff --git a/rpython/translator/stm/src_stm/stm/setup.c b/rpython/translator/stm/src_stm/stm/setup.c
--- a/rpython/translator/stm/src_stm/stm/setup.c
+++ b/rpython/translator/stm/src_stm/stm/setup.c
@@ -250,8 +250,6 @@
     set_gs_register(get_segment_base(num + 1));
     s_mutex_unlock();
 
-    DEBUG_EXPECT_SEGFAULT(true);
-
     if (num == 0) {
         dprintf(("STM_GC_NURSERY: %d\n", STM_GC_NURSERY));
         dprintf(("NB_PAGES: %d\n", NB_PAGES));
diff --git a/rpython/translator/stm/src_stm/stm/setup.h b/rpython/translator/stm/src_stm/stm/setup.h
--- a/rpython/translator/stm/src_stm/stm/setup.h
+++ b/rpython/translator/stm/src_stm/stm/setup.h
@@ -3,8 +3,8 @@
 static void setup_protection_settings(void);
 static pthread_t *_get_cpth(stm_thread_local_t *);
 #ifndef NDEBUG
-static __thread long _stm_segfault_expected = 0;
-#define DEBUG_EXPECT_SEGFAULT(v) do {if (v) _stm_segfault_expected++; else _stm_segfault_expected--;} while (0)
+static __thread long _stm_segfault_expected = 1;
+#define DEBUG_EXPECT_SEGFAULT(v) do {if (v) _stm_segfault_expected++; else _stm_segfault_expected--; assert(_stm_segfault_expected <= 1);} while (0)
 #else
 #define DEBUG_EXPECT_SEGFAULT(v) {}
 #endif
diff --git a/rpython/translator/stm/src_stm/stmgc.c b/rpython/translator/stm/src_stm/stmgc.c
--- a/rpython/translator/stm/src_stm/stmgc.c
+++ b/rpython/translator/stm/src_stm/stmgc.c
@@ -39,3 +39,4 @@
 #include "stm/prof.c"
 #include "stm/rewind_setjmp.c"
 #include "stm/finalizer.c"
+#include "stm/hashtable.c"
diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h
--- a/rpython/translator/stm/src_stm/stmgc.h
+++ b/rpython/translator/stm/src_stm/stmgc.h
@@ -196,10 +196,13 @@
         STM_SEGMENT->transaction_read_version;
 }
 
+#define _STM_WRITE_CHECK_SLOWPATH(obj)  \
+    UNLIKELY(((obj)->stm_flags & _STM_GCFLAG_WRITE_BARRIER) != 0)
+
 __attribute__((always_inline))
 static inline void stm_write(object_t *obj)
 {
-    if (UNLIKELY((obj->stm_flags & _STM_GCFLAG_WRITE_BARRIER) != 0))
+    if (_STM_WRITE_CHECK_SLOWPATH(obj))
         _stm_write_slowpath(obj);
 }
 
@@ -208,7 +211,7 @@
 static inline void stm_write_card(object_t *obj, uintptr_t index)
 {
     /* if GCFLAG_WRITE_BARRIER is set, then don't do anything more. */
-    if (UNLIKELY((obj->stm_flags & _STM_GCFLAG_WRITE_BARRIER) != 0)) {
+    if (_STM_WRITE_CHECK_SLOWPATH(obj)) {
 
         /* GCFLAG_WRITE_BARRIER is not set.  This might be because
            it's the first time we see a given small array; or it might
@@ -479,6 +482,38 @@
 /* dummies for now: */
 static inline void stm_flush_timing(stm_thread_local_t *tl, int verbose) {}
 
+
+/* Hashtables.  Keys are 64-bit unsigned integers, values are
+   'object_t *'.  Note that the type 'stm_hashtable_t' is not an
+   object type at all; you need to allocate and free it explicitly.
+   If you want to embed the hashtable inside an 'object_t' you
+   probably need a light finalizer to do the freeing. */
+typedef struct stm_hashtable_s stm_hashtable_t;
+typedef TLPREFIX struct stm_hashtable_entry_s stm_hashtable_entry_t;
+
+stm_hashtable_t *stm_hashtable_create(void);
+void stm_hashtable_free(stm_hashtable_t *);
+stm_hashtable_entry_t *stm_hashtable_lookup(object_t *, stm_hashtable_t *,
+                                            uintptr_t key);
+object_t *stm_hashtable_read(object_t *, stm_hashtable_t *, uintptr_t key);
+void stm_hashtable_write(object_t *, stm_hashtable_t *, uintptr_t key,
+                         object_t *nvalue, stm_thread_local_t *);
+void stm_hashtable_write_entry(object_t *hobj, stm_hashtable_entry_t *entry,
+                               object_t *nvalue);
+long stm_hashtable_length_upper_bound(stm_hashtable_t *);
+long stm_hashtable_list(object_t *, stm_hashtable_t *,
+                        stm_hashtable_entry_t **results);
+extern uint32_t stm_hashtable_entry_userdata;
+void stm_hashtable_tracefn(struct object_s *, stm_hashtable_t *,
+                           void (object_t **));
+
+struct stm_hashtable_entry_s {
+    struct object_s header;
+    uint32_t userdata;
+    uintptr_t index;
+    object_t *object;
+};
+
 /* ==================== END ==================== */
 
 static void (*stmcb_expand_marker)(char *segment_base, uintptr_t odd_number,

From noreply at buildbot.pypy.org  Thu Apr  9 17:12:29 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Thu,  9 Apr 2015 17:12:29 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8: import stmgc-c8 e55658d12179
Message-ID: <20150409151229.7A7B81C0173@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: stmgc-c8
Changeset: r76768:9789a066a768
Date: 2015-04-09 17:11 +0200
http://bitbucket.org/pypy/pypy/changeset/9789a066a768/

Log:	import stmgc-c8 e55658d12179

diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-1062987da64f
+e55658d12179
diff --git a/rpython/translator/stm/src_stm/stm/gcpage.c b/rpython/translator/stm/src_stm/stm/gcpage.c
--- a/rpython/translator/stm/src_stm/stm/gcpage.c
+++ b/rpython/translator/stm/src_stm/stm/gcpage.c
@@ -106,6 +106,9 @@
 object_t *_stm_allocate_old(ssize_t size_rounded_up)
 {
     /* only for tests xxx but stm_setup_prebuilt() uses this now too */
+    if (size_rounded_up <= GC_LAST_SMALL_SIZE)
+        return _stm_allocate_old_small(size_rounded_up);
+
     stm_char *p = allocate_outside_nursery_large(size_rounded_up);
     object_t *o = (object_t *)p;
 
diff --git a/rpython/translator/stm/src_stm/stm/hashtable.c b/rpython/translator/stm/src_stm/stm/hashtable.c
--- a/rpython/translator/stm/src_stm/stm/hashtable.c
+++ b/rpython/translator/stm/src_stm/stm/hashtable.c
@@ -109,6 +109,8 @@
 {
     /* can only be safely called during major GC, when all other threads
        are suspended */
+    assert(_has_mutex());
+
     long i;
     for (i = 1; i < NB_SEGMENTS; i++) {
         if (get_priv_segment(i)->transaction_state == TS_NONE)
@@ -174,6 +176,7 @@
         if (entry == NULL)
             continue;
         if (segment_base != NULL) {
+            /* -> compaction during major GC */
             if (((struct stm_hashtable_entry_s *)
                        REAL_ADDRESS(segment_base, entry))->object == NULL &&
                    !_stm_was_read_by_anybody((object_t *)entry)) {
@@ -283,6 +286,9 @@
         /* we can only enter here once!  If we allocate stuff, we may
            run the GC, and so 'hashtableobj' might move afterwards. */
         if (_is_in_nursery(hashtableobj)) {
+            /* this also means that the hashtable is from this
+               transaction and not visible to other segments yet, so
+               the new entry can be nursery-allocated. */
             entry = (stm_hashtable_entry_t *)
                 stm_allocate(sizeof(stm_hashtable_entry_t));
             entry->userdata = stm_hashtable_entry_userdata;

From noreply at buildbot.pypy.org  Fri Apr 10 09:53:44 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 10 Apr 2015 09:53:44 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix for issue #2019
Message-ID: <20150410075344.EDC181C0294@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76769:adfa94ce2ed9
Date: 2015-04-10 09:53 +0200
http://bitbucket.org/pypy/pypy/changeset/adfa94ce2ed9/

Log:	Fix for issue #2019

diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -501,12 +501,18 @@
         return SomeInteger(nonneg=True)
 
     def method_strip(self, chr=None):
+        if chr is None and isinstance(self, SomeUnicodeString):
+            raise AnnotatorError("unicode.strip() with no arg is not RPython")
         return self.basestringclass(no_nul=self.no_nul)
 
     def method_lstrip(self, chr=None):
+        if chr is None and isinstance(self, SomeUnicodeString):
+            raise AnnotatorError("unicode.lstrip() with no arg is not RPython")
         return self.basestringclass(no_nul=self.no_nul)
 
     def method_rstrip(self, chr=None):
+        if chr is None and isinstance(self, SomeUnicodeString):
+            raise AnnotatorError("unicode.rstrip() with no arg is not RPython")
         return self.basestringclass(no_nul=self.no_nul)
 
     def method_join(self, s_list):
diff --git a/rpython/rtyper/test/test_runicode.py b/rpython/rtyper/test/test_runicode.py
--- a/rpython/rtyper/test/test_runicode.py
+++ b/rpython/rtyper/test/test_runicode.py
@@ -306,3 +306,11 @@
         assert res == False
         res = self.interpret(f, [2])
         assert res == True
+
+    def test_strip_no_arg(self):
+
+        def f():
+            return u'abcdef'.strip()
+
+        e = py.test.raises(Exception, self.interpret, f, [])
+        assert "unicode.strip() with no arg is not RPython" in str(e.value)

From noreply at buildbot.pypy.org  Fri Apr 10 11:55:13 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 10 Apr 2015 11:55:13 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Add PLAN.
Message-ID: <20150410095513.16AF71C0294@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1683:d96ec16eaef6
Date: 2015-04-10 11:55 +0200
http://bitbucket.org/cffi/cffi/changeset/d96ec16eaef6/

Log:	Add PLAN.

diff --git a/new/PLAN b/new/PLAN
new file mode 100644
--- /dev/null
+++ b/new/PLAN
@@ -0,0 +1,114 @@
+
+==================================================
+CPython C extension module produced by recompile()
+==================================================
+
+Global variable:
+CTypeDescrObject *_cffi_types[];
+
+Every _cffi_types entry is initially an odd integer and is fixed
+to be a real `CTypeDescrObject *` later.
+
+The generated C functions are listed in _cffi_globals, a sorted array
+of entries which get turned lazily into real .  Each entry in this array has an index in the _cffi_types
+array, which describe the function type (CTOP_FUNCTION opcode, see
+below).  We turn the odd integers into real CTypeDescrObjects at the
+point where the entry is turned into a real builtin function object.
+
+The odd integers are "opcodes" that contain a type info in the lowest
+byte.  The remaining N-1 bytes of the integer is an "arg" that depends
+on the type info:
+
+CTOP_PRIMITIVE
+    the arg tells which primitive type it is
+
+CTOP_POINTER
+    the arg is the index of the item type in the _cffi_types array.
+
+CTOP_ARRAY
+    the arg is the index of the item type in the _cffi_types array.
+    followed by another opcode that contains (uintptr_t)length_of_array.
+
+CTOP_OPEN_ARRAY
+    for syntax like "int[]".  same as CTOP_ARRAY but without the length
+
+CTOP_STRUCT_UNION
+    the arg is the index of the struct/union in _cffi_structs_unions
+
+CTOP_ENUM
+    the arg is the index of the enum in _cffi_enums
+
+CTOP_TYPENAME
+    the arg is the index of the typename in _cffi_typenames
+
+CTOP_FUNCTION
+    the arg is the index of the result type in _cffi_types.
+    followed by other opcodes for the arguments.
+    terminated by CTOP_FUNCTION_END.
+
+CTOP_FUNCTION_END
+    the arg's lowest bit is set if there is a "..." argument.
+
+
+_cffi_globals
+-------------
+
+Lists global functions (with a type CTOP_FUNCTION), and also global
+variables (with a different type than CTOP_FUNCTION).
+
+struct {
+    const char *name;
+    void *address;
+    int type_index;
+}
+
+
+_cffi_structs_unions
+--------------------
+
+struct {
+    const char *name;
+    size_t size;
+    int alignment;
+    int flags;               // CT_UNION?
+    int num_fields;
+    int first_field_index;   // -> _cffi_fields array
+}
+
+struct _cffi_field_s {
+    const char *name;
+    size_t field_offset;
+    size_t field_size;
+    int field_bit_size;
+    int field_type;          // -> _cffi_types
+}
+
+
+_cffi_enums
+-----------
+
+struct {
+    const char *name;
+    int integer_type;        // -> _cffi_types
+}
+
+
+_cffi_constants
+---------------
+
+struct {
+    const char *name;
+    unsigned long long value;
+    int cinfo_or_type_index;    // CINFO_POSITIVE_INT, CINFO_NONPOSITIVE_INT
+}
+
+
+
+Runtime type parsing
+==================================================
+
+For ffi.new() etc.  This is done by turning the C declaration into an
+array of opcodes like above, and then turning these opcodes into real
+types.  The array of opcodes is then freed.  We use a cache from C
+declaration to final types.

From noreply at buildbot.pypy.org  Fri Apr 10 11:58:13 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 10 Apr 2015 11:58:13 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: add _cffi_typenames
Message-ID: <20150410095813.BB2D01C0294@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1684:bc6bab2c3d45
Date: 2015-04-10 11:58 +0200
http://bitbucket.org/cffi/cffi/changeset/bc6bab2c3d45/

Log:	add _cffi_typenames

diff --git a/new/PLAN b/new/PLAN
--- a/new/PLAN
+++ b/new/PLAN
@@ -94,6 +94,15 @@
 }
 
 
+_cffi_typenames
+---------------
+
+struct {
+    const char *name;
+    int type_index;          // -> _cffi_types
+}
+
+
 _cffi_constants
 ---------------
 

From noreply at buildbot.pypy.org  Fri Apr 10 15:37:28 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 10 Apr 2015 15:37:28 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: in-progress
Message-ID: <20150410133728.CD56A1C15B6@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1685:f53ba78aeffa
Date: 2015-04-10 15:38 +0200
http://bitbucket.org/cffi/cffi/changeset/f53ba78aeffa/

Log:	in-progress

diff --git a/new/PLAN b/new/PLAN
--- a/new/PLAN
+++ b/new/PLAN
@@ -71,7 +71,7 @@
     const char *name;
     size_t size;
     int alignment;
-    int flags;               // CT_UNION?
+    int flags;               // CT_UNION?  CT_IS_OPAQUE?
     int num_fields;
     int first_field_index;   // -> _cffi_fields array
 }
diff --git a/new/parse_c_type.c b/new/parse_c_type.c
--- a/new/parse_c_type.c
+++ b/new/parse_c_type.c
@@ -38,10 +38,12 @@
 };
 
 typedef struct {
+    struct _cffi_parse_info_s *info;
+    const char *p;
     enum token_e kind;
-    const char *p, **error_location, **error_message;
     size_t size;
-    ctype_opcode_t *opcodes, *opcodes_end;
+    _cffi_opcode_t *output;
+    unsigned long output_index;
 } token_t;
 
 static int is_space(char x)
@@ -153,54 +155,61 @@
     }
 }
 
-static void parse_error(token_t *tok, const char *msg)
+static int parse_error(token_t *tok, const char *msg)
 {
     if (tok->kind != TOK_ERROR) {
         tok->kind = TOK_ERROR;
-        if (tok->error_location)
-            *tok->error_location = tok->p;
-        if (tok->error_message)
-            *tok->error_message = msg;
+        if (tok->info->error_location)
+            *tok->info->error_location = tok->p;
+        if (tok->info->error_message)
+            *tok->info->error_message = msg;
     }
+    return -1;
 }
 
-static ctype_opcode_t *alloc_ds(token_t *tok, size_t num)
+static int write_ds(token_t *tok, _cffi_opcode_t ds)
 {
-    ctype_opcode_t *result = tok->opcodes;
-    if (num > tok->opcodes_end - result) {
-        parse_error(tok, "type too lengthy");
-        return NULL;
+    size_t index = tok->output_index;
+    if (index >= tok->info->output_size) {
+        parse_error(tok, "internal type complexity limit reached");
+        return -1;
     }
-    tok->opcodes += num;
-    return result;
+    tok->output[index] = ds;
+    tok->output_index = index + 1;
+    return index;
 }
 
-#if 0
-static void parse_complete(token_t *tok, _crx_qual_type *result);
+static int parse_complete(token_t *tok);
 
-static void parse_sequel(token_t *tok, intptr_t ds_end)
+static int parse_sequel(token_t *tok)
 {
-    intptr_t *ds;
-    while (tok->kind == TOK_STAR || tok->kind == TOK_CONST ||
-           tok->kind == TOK_VOLATILE) {
-        ds = alloc_ds(tok, 1);
-        if (ds == NULL)
-            return;
-        ds[0] = tok->kind;
+ header:
+    switch (tok->kind) {
+    case TOK_STAR:
+        write_ds(tok, _CFFI_OP(_CFFI_OP_POINTER, tok->output_index - 1));
         next_token(tok);
+        goto header;
+    case TOK_CONST:
+        /* ignored for now */
+        next_token(tok);
+        goto header;
+    case TOK_VOLATILE:
+        /* ignored for now */
+        next_token(tok);
+        goto header;
+    default:
+        break;
     }
 
+    _cffi_opcode_t slot_result = _CFFI_OP(0, tok->output_index - 1);
+    _cffi_opcode_t *p_current = &slot_result;
+
     int check_for_grouping = -1;
     if (tok->kind == TOK_IDENTIFIER) {
         next_token(tok);    /* skip a potential variable name */
         check_for_grouping = 1;
     }
 
-    intptr_t *jump_slot = alloc_ds(tok, 1);
-    if (jump_slot == NULL)
-        return;
-    *jump_slot = ds_end;
-
  next_right_part:
     check_for_grouping++;
 
@@ -209,6 +218,8 @@
     case TOK_OPEN_PAREN:
         next_token(tok);
 
+        abort();
+#if 0
         if (check_for_grouping == 0 && (tok->kind == TOK_STAR ||
                                         tok->kind == TOK_CONST ||
                                         tok->kind == TOK_VOLATILE ||
@@ -261,48 +272,43 @@
         }
         next_token(tok);
         goto next_right_part;
+#endif
 
     case TOK_OPEN_BRACKET:
     {
-        uintptr_t length = (uintptr_t)-1;
+        _cffi_opcode_t prev = *p_current;
+        *p_current = _CFFI_OP(_CFFI_GETOP(prev), tok->output_index);
+        p_current = &tok->output[tok->output_index];
+
         next_token(tok);
         if (tok->kind != TOK_CLOSE_BRACKET) {
-            if (tok->kind != TOK_INTEGER) {
-                parse_error(tok, "expected a positive integer constant");
-                return;
-            }
+            size_t length;
 
-            if (sizeof(uintptr_t) > sizeof(unsigned long))
+            if (tok->kind != TOK_INTEGER)
+                return parse_error(tok, "expected a positive integer constant");
+
+            if (sizeof(length) > sizeof(unsigned long))
                 length = strtoull(tok->p, NULL, 10);
             else
                 length = strtoul(tok->p, NULL, 10);
-            if (length == (uintptr_t)-1) {
-                parse_error(tok, "number too large");
-                return;
-            }
             next_token(tok);
+
+            write_ds(tok, _CFFI_OP(_CFFI_OP_ARRAY, _CFFI_GETARG(prev)));
+            write_ds(tok, (_cffi_opcode_t)length);
         }
+        else
+            write_ds(tok, _CFFI_OP(_CFFI_OP_OPEN_ARRAY, _CFFI_GETARG(prev)));
 
-        if (tok->kind != TOK_CLOSE_BRACKET) {
-            parse_error(tok, "expected ']'");
-            return;
-        }
+        if (tok->kind != TOK_CLOSE_BRACKET)
+            return parse_error(tok, "expected ']'");
         next_token(tok);
-
-        ds = alloc_ds(tok, 3);
-        if (ds == NULL)
-            return;
-        ds[0] = TOK_OPEN_BRACKET;
-        ds[1] = (intptr_t)length;
-        ds[2] = *jump_slot;
-        *jump_slot = -(ds - tok->all_delay_slots);
         goto next_right_part;
     }
     default:
         break;
     }
+    return _CFFI_GETARG(slot_result);
 }
-#endif
 
 #if 0
 static void fetch_delay_slots(token_t *tok, _crx_qual_type *result,
@@ -368,66 +374,55 @@
 }
 #endif
 
-static void parse_complete(token_t *tok)
+static int parse_complete(token_t *tok)
 {
-    int const_qualifier = 0, volatile_qualifier = 0;
-
  qualifiers:
     switch (tok->kind) {
     case TOK_CONST:
-        const_qualifier = 1;
+        /* ignored for now */
         next_token(tok);
         goto qualifiers;
     case TOK_VOLATILE:
-        volatile_qualifier = 1;
+        /* ignored for now */
         next_token(tok);
         goto qualifiers;
     default:
         ;
     }
 
-    int t1;
+    unsigned int t0;
+    _cffi_opcode_t t1;
     int modifiers_length = 0;
     int modifiers_sign = 0;
  modifiers:
     switch (tok->kind) {
 
     case TOK_SHORT:
-        if (modifiers_length != 0) {
-            parse_error(tok, "'short' after another 'short' or 'long'");
-            return;
-        }
+        if (modifiers_length != 0)
+            return parse_error(tok, "'short' after another 'short' or 'long'");
         modifiers_length--;
         next_token(tok);
         goto modifiers;
 
     case TOK_LONG:
-        if (modifiers_length < 0) {
-            parse_error(tok, "'long' after 'short'");
-            return;
-        }
-        if (modifiers_length >= 2) {
-            parse_error(tok, "'long long long' is too long");
-            return;
-        }
+        if (modifiers_length < 0)
+            return parse_error(tok, "'long' after 'short'");
+        if (modifiers_length >= 2)
+            return parse_error(tok, "'long long long' is too long");
         modifiers_length++;
         next_token(tok);
         goto modifiers;
 
     case TOK_SIGNED:
-        if (modifiers_sign) {
-            parse_error(tok, "multiple 'signed' or 'unsigned'");
-            return;
-        }
+        if (modifiers_sign)
+            return parse_error(tok, "multiple 'signed' or 'unsigned'");
         modifiers_sign++;
         next_token(tok);
         goto modifiers;
 
     case TOK_UNSIGNED:
-        if (modifiers_sign) {
-            parse_error(tok, "multiple 'signed' or 'unsigned'");
-            return;
-        }
+        if (modifiers_sign)
+            return parse_error(tok, "multiple 'signed' or 'unsigned'");
         modifiers_sign--;
         next_token(tok);
         goto modifiers;
@@ -445,23 +440,18 @@
         case TOK_FLOAT:
         case TOK_STRUCT:
         case TOK_UNION:
-            parse_error(tok, "invalid combination of types");
-            return;
+            return parse_error(tok, "invalid combination of types");
 
         case TOK_DOUBLE:
-            if (modifiers_sign != 0 || modifiers_length != 1) {
-                parse_error(tok, "invalid combination of types");
-                return;
-            }
+            if (modifiers_sign != 0 || modifiers_length != 1)
+                return parse_error(tok, "invalid combination of types");
             next_token(tok);
-            t1 = CTOP_LONGDOUBLE;
+            t0 = _CFFI_PRIM_LONGDOUBLE;
             break;
 
         case TOK_CHAR:
-            if (modifiers_length != 0) {
-                parse_error(tok, "invalid combination of types");
-                return;
-            }
+            if (modifiers_length != 0)
+                return parse_error(tok, "invalid combination of types");
             modifiers_length = -2;
             /* fall-through */
         case TOK_INT:
@@ -470,41 +460,42 @@
         default:
             if (modifiers_sign >= 0)
                 switch (modifiers_length) {
-                case -2: t1 = CTOP_SCHAR; break;
-                case -1: t1 = CTOP_SHORT; break;
-                case 1:  t1 = CTOP_LONG; break;
-                case 2:  t1 = CTOP_LONGLONG; break;
-                default: t1 = CTOP_INT; break;
+                case -2: t0 = _CFFI_PRIM_SCHAR; break;
+                case -1: t0 = _CFFI_PRIM_SHORT; break;
+                case 1:  t0 = _CFFI_PRIM_LONG; break;
+                case 2:  t0 = _CFFI_PRIM_LONGLONG; break;
+                default: t0 = _CFFI_PRIM_INT; break;
                 }
             else
                 switch (modifiers_length) {
-                case -2: t1 = CTOP_UCHAR; break;
-                case -1: t1 = CTOP_USHORT; break;
-                case 1:  t1 = CTOP_ULONG; break;
-                case 2:  t1 = CTOP_ULONGLONG; break;
-                default: t1 = CTOP_UINT; break;
+                case -2: t0 = _CFFI_PRIM_UCHAR; break;
+                case -1: t0 = _CFFI_PRIM_USHORT; break;
+                case 1:  t0 = _CFFI_PRIM_ULONG; break;
+                case 2:  t0 = _CFFI_PRIM_ULONGLONG; break;
+                default: t0 = _CFFI_PRIM_UINT; break;
                 }
         }
+        t1 = _CFFI_OP(_CFFI_OP_PRIMITIVE, t0);
     }
     else {
         switch (tok->kind) {
         case TOK_INT:
-            t1 = CTOP_INT;
+            t1 = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_INT);
             break;
         case TOK_CHAR:
-            t1 = CTOP_CHAR;
+            t1 = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_CHAR);
             break;
         case TOK_VOID:
-            t1 = CTOP_VOID;
+            t1 = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_VOID);
             break;
         case TOK__BOOL:
-            t1 = CTOP_BOOL;
+            t1 = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_BOOL);
             break;
         case TOK_FLOAT:
-            t1 = CTOP_FLOAT;
+            t1 = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_FLOAT);
             break;
         case TOK_DOUBLE:
-            t1 = CTOP_DOUBLE;
+            t1 = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_DOUBLE);
             break;
         case TOK_IDENTIFIER:
         {
@@ -550,41 +541,32 @@
 #endif
         }
         default:
-            parse_error(tok, "identifier expected");
-            return;
+            return parse_error(tok, "identifier expected");
         }
         next_token(tok);
     }
-    *alloc_ds(tok, 1) = t1;
-    if (const_qualifier)
-        *alloc_ds(tok, 1) = CTOP_CONST;
-    if (volatile_qualifier)
-        *alloc_ds(tok, 1) = CTOP_VOLATILE;
 
-    //parse_sequel(tok, CTOP_END);
-    *alloc_ds(tok, 1) = CTOP_END;
+    write_ds(tok, t1);
+    return parse_sequel(tok);
 }
 
 
-int parse_c_type(const char *input,
-                 ctype_opcode_t *output, size_t output_size,
-                 const char **error_loc, const char **error_msg)
+int parse_c_type(struct _cffi_parse_info_s *info, const char *input)
 {
+    int result;
     token_t token;
 
+    token.info = info;
     token.kind = TOK_START;
     token.p = input;
-    token.error_location = error_loc;
-    token.error_message = error_msg;
     token.size = 0;
-    token.opcodes = output;
-    token.opcodes_end = output + output_size;
+    token.output = info->output;
+    token.output_index = 0;
+
     next_token(&token);
-    parse_complete(&token);
+    result = parse_complete(&token);
 
-    if (token.kind != TOK_END) {
-        parse_error(&token, "unexpected symbol");
-        return -1;
-    }
-    return 0;
+    if (token.kind != TOK_END)
+        return parse_error(&token, "unexpected symbol");
+    return result;
 }
diff --git a/new/parse_c_type.h b/new/parse_c_type.h
--- a/new/parse_c_type.h
+++ b/new/parse_c_type.h
@@ -1,29 +1,100 @@
 
 
-typedef int ctype_opcode_t;
+typedef void *_cffi_opcode_t;
 
-#define CTOP_END         0
-#define CTOP_CONST       1
-#define CTOP_VOLATILE    2
+#define _CFFI_OP(opcode, arg)   (_cffi_opcode_t)(opcode | (((unsigned long)(arg)) << 8))
+#define _CFFI_GETOP(cffi_opcode)    ((unsigned char)(unsigned long)cffi_opcode)
+#define _CFFI_GETARG(cffi_opcode)   (((unsigned long)cffi_opcode) >> 8)
 
-#define CTOP_VOID        100
-#define CTOP_BOOL        101
-#define CTOP_CHAR        102
-#define CTOP_SCHAR       103
-#define CTOP_UCHAR       104
-#define CTOP_SHORT       105
-#define CTOP_USHORT      106
-#define CTOP_INT         107
-#define CTOP_UINT        108
-#define CTOP_LONG        109
-#define CTOP_ULONG       110
-#define CTOP_LONGLONG    111
-#define CTOP_ULONGLONG   112
-#define CTOP_FLOAT       113
-#define CTOP_DOUBLE      114
-#define CTOP_LONGDOUBLE  115
+#define _CFFI_OP_PRIMITIVE       1
+#define _CFFI_OP_POINTER         3
+#define _CFFI_OP_ARRAY           5
+#define _CFFI_OP_OPEN_ARRAY      7
+#define _CFFI_OP_STRUCT_UNION    9
+#define _CFFI_OP_ENUM           11
+#define _CFFI_OP_TYPENAME       13
+#define _CFFI_OP_FUNCTION       15
+#define _CFFI_OP_FUNCTION_END   17
 
+#define _CFFI_PRIM_VOID          0
+#define _CFFI_PRIM_BOOL          1
+#define _CFFI_PRIM_CHAR          2
+#define _CFFI_PRIM_SCHAR         3
+#define _CFFI_PRIM_UCHAR         4
+#define _CFFI_PRIM_SHORT         5
+#define _CFFI_PRIM_USHORT        6
+#define _CFFI_PRIM_INT           7
+#define _CFFI_PRIM_UINT          8
+#define _CFFI_PRIM_LONG          9
+#define _CFFI_PRIM_ULONG        10
+#define _CFFI_PRIM_LONGLONG     11
+#define _CFFI_PRIM_ULONGLONG    12
+#define _CFFI_PRIM_FLOAT        13
+#define _CFFI_PRIM_DOUBLE       14
+#define _CFFI_PRIM_LONGDOUBLE   15
 
-int parse_c_type(const char *input,
-                 ctype_opcode_t *output, size_t output_size,
-                 const char **error_loc, const char **error_msg);
+
+struct _cffi_global_s {
+    const char *name;
+    void *address;
+    int type_index;
+};
+
+struct _cffi_constant_s {
+    const char *name;
+    unsigned long long value;
+    int type_index_or_plain;
+};
+#define _CFFI_PLAIN_POSITIVE_INT     (-1)
+#define _CFFI_PLAIN_NONPOSITIVE_INT  (-2)
+
+struct _cffi_struct_union_s {
+    const char *name;
+    size_t size;
+    int alignment;
+    int flags;               // CT_UNION?  CT_IS_OPAQUE?
+    int num_fields;
+    int first_field_index;   // -> _cffi_fields array
+};
+
+struct _cffi_field_s {
+    const char *name;
+    size_t field_offset;
+    size_t field_size;
+    int field_bit_size;
+    int field_type;          // -> _cffi_types
+};
+
+struct _cffi_enum_s {
+    const char *name;
+    int integer_type;        // -> _cffi_types
+};
+
+struct _cffi_typename_s {
+    const char *name;
+    int type_index;          // -> _cffi_types
+};
+
+struct _cffi_type_context_s {
+    const struct _cffi_global_s *globals;
+    const struct _cffi_constant_s *constants;
+    const struct _cffi_struct_union_s *structs_unions;
+    const struct _cffi_field_s *fields;
+    const struct _cffi_enum_s *enums;
+    const struct _cffi_typename_s *typenames;
+    int num_globals;
+    int num_constants;
+    int num_structs_unions;
+    int num_enums;
+    int num_typenames;
+};
+
+struct _cffi_parse_info_s {
+    struct _cffi_type_context_s *ctx;
+    _cffi_opcode_t *output;
+    int output_size;
+    const char **error_location;
+    const char **error_message;
+};
+
+int parse_c_type(struct _cffi_parse_info_s *info, const char *input);
diff --git a/new/test_parse_c_type.py b/new/test_parse_c_type.py
--- a/new/test_parse_c_type.py
+++ b/new/test_parse_c_type.py
@@ -1,53 +1,77 @@
+import re
 import os
 import cffi
 
+r_macro = re.compile(r"#define \w+[(][^\n]*")
+r_define = re.compile(r"(#define \w+) [^\n]*")
+header = open('parse_c_type.h').read()
+header = r_macro.sub(r"", header)
+header = r_define.sub(r"\1 ...", header)
+
 ffi = cffi.FFI()
-ffi.cdef("""
-typedef int ctype_opcode_t;
-
-#define CTOP_END         ...
-#define CTOP_CONST       ...
-#define CTOP_VOLATILE    ...
-
-#define CTOP_VOID        ...
-#define CTOP_BOOL        ...
-#define CTOP_CHAR        ...
-#define CTOP_SCHAR       ...
-#define CTOP_UCHAR       ...
-#define CTOP_SHORT       ...
-#define CTOP_USHORT      ...
-#define CTOP_INT         ...
-#define CTOP_UINT        ...
-#define CTOP_LONG        ...
-#define CTOP_ULONG       ...
-#define CTOP_LONGLONG    ...
-#define CTOP_ULONGLONG   ...
-#define CTOP_FLOAT       ...
-#define CTOP_DOUBLE      ...
-#define CTOP_LONGDOUBLE  ...
-
-int parse_c_type(const char *input,
-                 ctype_opcode_t *output, size_t output_size,
-                 const char **error_loc, const char **error_msg);
-""")
+ffi.cdef(header)
 
 lib = ffi.verify(open('parse_c_type.c').read(),
                  include_dirs=[os.getcwd()])
 
+class ParseError(Exception):
+    pass
+
+def parse(input):
+    out = ffi.new("_cffi_opcode_t[]", 100)
+    info = ffi.new("struct _cffi_parse_info_s *")
+    info.output = out
+    info.output_size = len(out)
+    for j in range(len(out)):
+        out[j] = ffi.cast("void *", -424242)
+    c_input = ffi.new("char[]", input)
+    res = lib.parse_c_type(info, c_input)
+    if res < 0:
+        raise ParseError(ffi.string(info.error_message),
+                         ffi.string(info.error_location) - c_input)
+    assert 0 <= res < len(out)
+    result = []
+    for j in range(len(out)):
+        if out[j] == ffi.cast("void *", -424242):
+            assert res < j
+            break
+        i = int(ffi.cast("intptr_t", out[j]))
+        if j == res:
+            result.append('->')
+        result.append(i)
+    return result
+
+def make_getter(name):
+    opcode = getattr(lib, '_CFFI_OP_' + name)
+    def getter(value):
+        return opcode | (value << 8)
+    return getter
+
+Prim = make_getter('PRIMITIVE')
+Array = make_getter('ARRAY')
+OpenArray = make_getter('OPEN_ARRAY')
+
 
 def test_simple():
-    out = ffi.new("ctype_opcode_t[]", 100)
     for simple_type, expected in [
-            ("int", lib.CTOP_INT),
-            ("signed int", lib.CTOP_INT),
-            ("  long  ", lib.CTOP_LONG),
-            ("long int", lib.CTOP_LONG),
-            ("unsigned short", lib.CTOP_USHORT),
-            ("long double", lib.CTOP_LONGDOUBLE),
+            ("int", lib._CFFI_PRIM_INT),
+            ("signed int", lib._CFFI_PRIM_INT),
+            ("  long  ", lib._CFFI_PRIM_LONG),
+            ("long int", lib._CFFI_PRIM_LONG),
+            ("unsigned short", lib._CFFI_PRIM_USHORT),
+            ("long double", lib._CFFI_PRIM_LONGDOUBLE),
             ]:
-        for j in range(len(out)):
-            out[j] = -424242
-        res = lib.parse_c_type(simple_type, out, 100, ffi.NULL, ffi.NULL)
-        assert res == 0
-        assert out[0] == expected
-        assert out[1] == lib.CTOP_END
+        assert parse(simple_type) == ['->', Prim(expected)]
+
+def test_array():
+    assert parse("int[5]") == [Prim(lib._CFFI_PRIM_INT), '->', Array(0), 5]
+    assert parse("int[]") == [Prim(lib._CFFI_PRIM_INT), '->', OpenArray(0)]
+    assert parse("int[5][8]") == [Prim(lib._CFFI_PRIM_INT),
+                                  '->', Array(3),
+                                  5,
+                                  Array(0),
+                                  8]
+    assert parse("int[][8]") == [Prim(lib._CFFI_PRIM_INT),
+                                  '->', OpenArray(2),
+                                  Array(0),
+                                  8]

From noreply at buildbot.pypy.org  Fri Apr 10 18:04:48 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 10 Apr 2015 18:04:48 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: in-progress: simplify my life with
 adding the occasional OP_NOOP
Message-ID: <20150410160448.63B351C01F3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1686:5dd093128d4b
Date: 2015-04-10 18:05 +0200
http://bitbucket.org/cffi/cffi/changeset/5dd093128d4b/

Log:	in-progress: simplify my life with adding the occasional OP_NOOP

diff --git a/new/parse_c_type.c b/new/parse_c_type.c
--- a/new/parse_c_type.c
+++ b/new/parse_c_type.c
@@ -181,12 +181,20 @@
 
 static int parse_complete(token_t *tok);
 
-static int parse_sequel(token_t *tok)
+static int parse_sequel(token_t *tok, int outer)
 {
+    /* Emit opcodes for the "sequel", which is the optional part of a
+       type declaration that follows the type name, i.e. everything
+       with '*', '[ ]', '( )'.  Returns the entry point index pointing
+       the innermost opcode (the one that corresponds to the complete
+       type).  The 'outer' argument is the index of the opcode outside
+       this "sequel".
+     */
+
  header:
     switch (tok->kind) {
     case TOK_STAR:
-        write_ds(tok, _CFFI_OP(_CFFI_OP_POINTER, tok->output_index - 1));
+        outer = write_ds(tok, _CFFI_OP(_CFFI_OP_POINTER, outer));
         next_token(tok);
         goto header;
     case TOK_CONST:
@@ -201,35 +209,36 @@
         break;
     }
 
-    _cffi_opcode_t slot_result = _CFFI_OP(0, tok->output_index - 1);
-    _cffi_opcode_t *p_current = &slot_result;
-
-    int check_for_grouping = -1;
+    int check_for_grouping = 1;
     if (tok->kind == TOK_IDENTIFIER) {
         next_token(tok);    /* skip a potential variable name */
-        check_for_grouping = 1;
+        check_for_grouping = 0;
     }
 
- next_right_part:
-    check_for_grouping++;
+    _cffi_opcode_t result = 0;
+    _cffi_opcode_t *p_current = &result;
 
-    switch (tok->kind) {
-
-    case TOK_OPEN_PAREN:
+    while (tok->kind == TOK_OPEN_PAREN) {
         next_token(tok);
 
-        abort();
-#if 0
-        if (check_for_grouping == 0 && (tok->kind == TOK_STAR ||
-                                        tok->kind == TOK_CONST ||
-                                        tok->kind == TOK_VOLATILE ||
-                                        tok->kind == TOK_OPEN_BRACKET)) {
-            /* just parentheses for grouping */
-            ds = tok->delay_slots;
-            parse_sequel(tok, *jump_slot);
-            *jump_slot = -(ds - tok->all_delay_slots);
+        if ((check_for_grouping--) == 1 && (tok->kind == TOK_STAR ||
+                                            tok->kind == TOK_CONST ||
+                                            tok->kind == TOK_VOLATILE ||
+                                            tok->kind == TOK_OPEN_BRACKET)) {
+            /* just parentheses for grouping.  Use a OP_NOOP to simplify */
+            int x;
+            assert(p_current == &result);
+            x = tok->output_index;
+            p_current = tok->output + x;
+
+            write_ds(tok, _CFFI_OP(_CFFI_OP_NOOP, 0));
+
+            x = parse_sequel(tok, x);
+            result = _CFFI_OP(_CFFI_GETOP(0), x);
         }
         else {
+            abort();
+#if 0
             /* function type */
             ds = alloc_ds(tok, 2);
             if (ds == NULL)
@@ -264,21 +273,17 @@
             assert(ds_next == ds + 2 + 2 * ds[1]);
             *ds_next = *jump_slot;
             *jump_slot = -(ds - tok->all_delay_slots);
+#endif
         }
 
-        if (tok->kind != TOK_CLOSE_PAREN) {
-            parse_error(tok, "expected ')'");
-            return;
-        }
+        if (tok->kind != TOK_CLOSE_PAREN)
+            return parse_error(tok, "expected ')'");
         next_token(tok);
-        goto next_right_part;
-#endif
+    }
 
-    case TOK_OPEN_BRACKET:
-    {
-        _cffi_opcode_t prev = *p_current;
-        *p_current = _CFFI_OP(_CFFI_GETOP(prev), tok->output_index);
-        p_current = &tok->output[tok->output_index];
+    while (tok->kind == TOK_OPEN_BRACKET) {
+        *p_current = _CFFI_OP(_CFFI_GETOP(*p_current), tok->output_index);
+        p_current = tok->output + tok->output_index;
 
         next_token(tok);
         if (tok->kind != TOK_CLOSE_BRACKET) {
@@ -293,21 +298,19 @@
                 length = strtoul(tok->p, NULL, 10);
             next_token(tok);
 
-            write_ds(tok, _CFFI_OP(_CFFI_OP_ARRAY, _CFFI_GETARG(prev)));
+            write_ds(tok, _CFFI_OP(_CFFI_OP_ARRAY, 0));
             write_ds(tok, (_cffi_opcode_t)length);
         }
         else
-            write_ds(tok, _CFFI_OP(_CFFI_OP_OPEN_ARRAY, _CFFI_GETARG(prev)));
+            write_ds(tok, _CFFI_OP(_CFFI_OP_OPEN_ARRAY, 0));
 
         if (tok->kind != TOK_CLOSE_BRACKET)
             return parse_error(tok, "expected ']'");
         next_token(tok);
-        goto next_right_part;
     }
-    default:
-        break;
-    }
-    return _CFFI_GETARG(slot_result);
+
+    *p_current = _CFFI_OP(_CFFI_GETOP(*p_current), outer);
+    return _CFFI_GETARG(result);
 }
 
 #if 0
@@ -546,8 +549,7 @@
         next_token(tok);
     }
 
-    write_ds(tok, t1);
-    return parse_sequel(tok);
+    return parse_sequel(tok, write_ds(tok, t1));
 }
 
 
diff --git a/new/parse_c_type.h b/new/parse_c_type.h
--- a/new/parse_c_type.h
+++ b/new/parse_c_type.h
@@ -15,6 +15,7 @@
 #define _CFFI_OP_TYPENAME       13
 #define _CFFI_OP_FUNCTION       15
 #define _CFFI_OP_FUNCTION_END   17
+#define _CFFI_OP_NOOP           19
 
 #define _CFFI_PRIM_VOID          0
 #define _CFFI_PRIM_BOOL          1
diff --git a/new/test_parse_c_type.py b/new/test_parse_c_type.py
--- a/new/test_parse_c_type.py
+++ b/new/test_parse_c_type.py
@@ -48,8 +48,10 @@
     return getter
 
 Prim = make_getter('PRIMITIVE')
+Pointer = make_getter('POINTER')
 Array = make_getter('ARRAY')
 OpenArray = make_getter('OPEN_ARRAY')
+NoOp = make_getter('NOOP')
 
 
 def test_simple():
@@ -72,6 +74,30 @@
                                   Array(0),
                                   8]
     assert parse("int[][8]") == [Prim(lib._CFFI_PRIM_INT),
-                                  '->', OpenArray(2),
-                                  Array(0),
-                                  8]
+                                 '->', OpenArray(2),
+                                 Array(0),
+                                 8]
+
+def test_pointer():
+    assert parse("int*") == [Prim(lib._CFFI_PRIM_INT), '->', Pointer(0)]
+    assert parse("int***") == [Prim(lib._CFFI_PRIM_INT),
+                               Pointer(0), Pointer(1), '->', Pointer(2)]
+
+def test_grouping():
+    assert parse("int*[]") == [Prim(lib._CFFI_PRIM_INT),
+                               Pointer(0), '->', OpenArray(1)]
+    assert parse("int**[][8]") == [Prim(lib._CFFI_PRIM_INT),
+                                   Pointer(0), Pointer(1),
+                                   '->', OpenArray(4), Array(2), 8]
+    assert parse("int(*)[]") == [Prim(lib._CFFI_PRIM_INT),
+                                 NoOp(3), '->', Pointer(1), OpenArray(0)]
+    assert parse("int(*)[][8]") == [Prim(lib._CFFI_PRIM_INT),
+                                    NoOp(3), '->', Pointer(1),
+                                    OpenArray(4), Array(0), 8]
+    assert parse("int**(**)") == [Prim(lib._CFFI_PRIM_INT),
+                                  Pointer(0), Pointer(1),
+                                  NoOp(2), Pointer(3), '->', Pointer(4)]
+    assert parse("int**(**)[]") == [Prim(lib._CFFI_PRIM_INT),
+                                    Pointer(0), Pointer(1),
+                                    NoOp(6), Pointer(3), '->', Pointer(4),
+                                    OpenArray(2)]

From noreply at buildbot.pypy.org  Fri Apr 10 18:45:03 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 10 Apr 2015 18:45:03 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Function types
Message-ID: <20150410164503.6F8881C01F3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1687:db54ee477ad5
Date: 2015-04-10 18:45 +0200
http://bitbucket.org/cffi/cffi/changeset/db54ee477ad5/

Log:	Function types

diff --git a/new/parse_c_type.c b/new/parse_c_type.c
--- a/new/parse_c_type.c
+++ b/new/parse_c_type.c
@@ -77,6 +77,22 @@
     return *p;
 }
 
+static int number_of_commas(token_t *tok)
+{
+    const char *p = tok->p;
+    int result = 0;
+    int nesting = 0;
+    while (1) {
+        switch (*p++) {
+        case ',': result++; break;
+        case '(': nesting++; break;
+        case ')': if ((--nesting) < 0) return result; break;
+        case 0:   return result;
+        default:  break;
+        }
+    }
+}
+
 static void next_token(token_t *tok)
 {
     const char *p = tok->p + tok->size;
@@ -237,43 +253,47 @@
             result = _CFFI_OP(_CFFI_GETOP(0), x);
         }
         else {
-            abort();
-#if 0
             /* function type */
-            ds = alloc_ds(tok, 2);
-            if (ds == NULL)
-                return;
-            ds[0] = TOK_OPEN_PAREN;
-            ds[1] = 0;
+            int arg_total, base_index, arg_next, has_ellipsis=0;
+
             if (tok->kind == TOK_VOID && get_following_char(tok) == ')') {
                 next_token(tok);
             }
+
+            /* (over-)estimate 'arg_total'.  May return 1 when it is really 0 */
+            arg_total = number_of_commas(tok) + 1;
+
+            *p_current = _CFFI_OP(_CFFI_GETOP(*p_current), tok->output_index);
+            p_current = tok->output + tok->output_index;
+
+            base_index = write_ds(tok, _CFFI_OP(_CFFI_OP_FUNCTION, 0));
+            if (base_index < 0)
+                return -1;
+            /* reserve (arg_total + 1) slots for the arguments and the
+               final FUNCTION_END */
+            for (arg_next = 0; arg_next <= arg_total; arg_next++)
+                if (write_ds(tok, _CFFI_OP(0, 0)) < 0)
+                    return -1;
+
+            arg_next = base_index + 1;
+
             if (tok->kind != TOK_CLOSE_PAREN) {
                 while (1) {
                     if (tok->kind == TOK_DOTDOTDOT) {
-                        ds[0] = TOK_DOTDOTDOT;
+                        has_ellipsis = 1;
                         next_token(tok);
                         break;
                     }
-                    intptr_t *ds_type = alloc_ds(tok, 2);
-                    if (ds_type == NULL)
-                        return;
-                    assert(ds_type == ds + 2 + 2 * ds[1]);
-                    assert(2 * sizeof(intptr_t) >= sizeof(_crx_qual_type));
-                    parse_complete(tok, (_crx_qual_type *)ds_type);
-                    ds[1]++;
+                    int arg = parse_complete(tok);
+                    assert(arg_next - base_index <= arg_total);
+                    tok->output[arg_next++] = _CFFI_OP(_CFFI_OP_NOOP, arg);
                     if (tok->kind != TOK_COMMA)
                         break;
                     next_token(tok);
                 }
             }
-            intptr_t *ds_next = alloc_ds(tok, 1);
-            if (ds_next == NULL)
-                return;
-            assert(ds_next == ds + 2 + 2 * ds[1]);
-            *ds_next = *jump_slot;
-            *jump_slot = -(ds - tok->all_delay_slots);
-#endif
+            tok->output[arg_next] = _CFFI_OP(_CFFI_OP_FUNCTION_END,
+                                             has_ellipsis);
         }
 
         if (tok->kind != TOK_CLOSE_PAREN)
diff --git a/new/test_parse_c_type.py b/new/test_parse_c_type.py
--- a/new/test_parse_c_type.py
+++ b/new/test_parse_c_type.py
@@ -52,6 +52,8 @@
 Array = make_getter('ARRAY')
 OpenArray = make_getter('OPEN_ARRAY')
 NoOp = make_getter('NOOP')
+Func = make_getter('FUNCTION')
+FuncEnd = make_getter('FUNCTION_END')
 
 
 def test_simple():
@@ -101,3 +103,25 @@
                                     Pointer(0), Pointer(1),
                                     NoOp(6), Pointer(3), '->', Pointer(4),
                                     OpenArray(2)]
+
+def test_simple_function():
+    assert parse("int()") == [Prim(lib._CFFI_PRIM_INT),
+                              '->', Func(0), FuncEnd(0), 0]
+    assert parse("int(int)") == [Prim(lib._CFFI_PRIM_INT),
+                                 '->', Func(0), NoOp(4), FuncEnd(0),
+                                 Prim(lib._CFFI_PRIM_INT)]
+    assert parse("int(long, char)") == [
+                                 Prim(lib._CFFI_PRIM_INT),
+                                 '->', Func(0), NoOp(5), NoOp(6), FuncEnd(0),
+                                 Prim(lib._CFFI_PRIM_LONG),
+                                 Prim(lib._CFFI_PRIM_CHAR)]
+    assert parse("int(int*)") == [Prim(lib._CFFI_PRIM_INT),
+                                  '->', Func(0), NoOp(5), FuncEnd(0),
+                                  Prim(lib._CFFI_PRIM_INT),
+                                  Pointer(4)]
+    assert parse("int*(void)") == [Prim(lib._CFFI_PRIM_INT),
+                                   Pointer(0),
+                                   '->', Func(1), FuncEnd(0), 0]
+    assert parse("int(int, ...)") == [Prim(lib._CFFI_PRIM_INT),
+                                      '->', Func(0), NoOp(5), FuncEnd(1), 0,
+                                      Prim(lib._CFFI_PRIM_INT)]

From noreply at buildbot.pypy.org  Fri Apr 10 19:00:22 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 10 Apr 2015 19:00:22 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: more tests, one fix
Message-ID: <20150410170022.E0B351C0294@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1688:739312aad65c
Date: 2015-04-10 19:00 +0200
http://bitbucket.org/cffi/cffi/changeset/739312aad65c/

Log:	more tests, one fix

diff --git a/new/parse_c_type.c b/new/parse_c_type.c
--- a/new/parse_c_type.c
+++ b/new/parse_c_type.c
@@ -84,7 +84,7 @@
     int nesting = 0;
     while (1) {
         switch (*p++) {
-        case ',': result++; break;
+        case ',': result += !nesting; break;
         case '(': nesting++; break;
         case ')': if ((--nesting) < 0) return result; break;
         case 0:   return result;
diff --git a/new/test_parse_c_type.py b/new/test_parse_c_type.py
--- a/new/test_parse_c_type.py
+++ b/new/test_parse_c_type.py
@@ -41,6 +41,14 @@
         result.append(i)
     return result
 
+def parsex(input):
+    result = parse(input)
+    def str_if_int(x):
+        if isinstance(x, str):
+            return x
+        return '%d,%d' % (x & 255, x >> 8)
+    return '  '.join(map(str_if_int, result))
+
 def make_getter(name):
     opcode = getattr(lib, '_CFFI_OP_' + name)
     def getter(value):
@@ -125,3 +133,20 @@
     assert parse("int(int, ...)") == [Prim(lib._CFFI_PRIM_INT),
                                       '->', Func(0), NoOp(5), FuncEnd(1), 0,
                                       Prim(lib._CFFI_PRIM_INT)]
+
+def test_internal_function():
+    assert parse("int(*)()") == [Prim(lib._CFFI_PRIM_INT),
+                                 NoOp(3), '->', Pointer(1),
+                                 Func(0), FuncEnd(0), 0]
+    assert parse("int(*())[]") == [Prim(lib._CFFI_PRIM_INT),
+                                   NoOp(6), Pointer(1),
+                                   '->', Func(2), FuncEnd(0), 0,
+                                   OpenArray(0)]
+    assert parse("int(char(*)(long, short))") == [
+        Prim(lib._CFFI_PRIM_INT),
+        '->', Func(0), NoOp(6), FuncEnd(0),
+        Prim(lib._CFFI_PRIM_CHAR),
+        NoOp(7), Pointer(5),
+        Func(4), NoOp(11), NoOp(12), FuncEnd(0),
+        Prim(lib._CFFI_PRIM_LONG),
+        Prim(lib._CFFI_PRIM_SHORT)]

From noreply at buildbot.pypy.org  Fri Apr 10 19:06:16 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 10 Apr 2015 19:06:16 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Fix the argument types (function ->
 ptr_to_function; array -> pointer)
Message-ID: <20150410170616.187991C0294@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1689:95d687f8e1bd
Date: 2015-04-10 19:06 +0200
http://bitbucket.org/cffi/cffi/changeset/95d687f8e1bd/

Log:	Fix the argument types (function -> ptr_to_function; array ->
	pointer)

diff --git a/new/parse_c_type.c b/new/parse_c_type.c
--- a/new/parse_c_type.c
+++ b/new/parse_c_type.c
@@ -285,8 +285,21 @@
                         break;
                     }
                     int arg = parse_complete(tok);
+                    _cffi_opcode_t oarg;
                     assert(arg_next - base_index <= arg_total);
-                    tok->output[arg_next++] = _CFFI_OP(_CFFI_OP_NOOP, arg);
+                    switch (_CFFI_GETOP(tok->output[arg])) {
+                    case _CFFI_OP_ARRAY:
+                    case _CFFI_OP_OPEN_ARRAY:
+                        arg = _CFFI_GETARG(tok->output[arg]);
+                        /* fall-through */
+                    case _CFFI_OP_FUNCTION:
+                        oarg = _CFFI_OP(_CFFI_OP_POINTER, arg);
+                        break;
+                    default:
+                        oarg = _CFFI_OP(_CFFI_OP_NOOP, arg);
+                        break;
+                    }
+                    tok->output[arg_next++] = oarg;
                     if (tok->kind != TOK_COMMA)
                         break;
                     next_token(tok);
diff --git a/new/test_parse_c_type.py b/new/test_parse_c_type.py
--- a/new/test_parse_c_type.py
+++ b/new/test_parse_c_type.py
@@ -150,3 +150,17 @@
         Func(4), NoOp(11), NoOp(12), FuncEnd(0),
         Prim(lib._CFFI_PRIM_LONG),
         Prim(lib._CFFI_PRIM_SHORT)]
+
+def test_fix_arg_types():
+    assert parse("int(char(long, short))") == [
+        Prim(lib._CFFI_PRIM_INT),
+        '->', Func(0), Pointer(5), FuncEnd(0),
+        Prim(lib._CFFI_PRIM_CHAR),
+        Func(4), NoOp(9), NoOp(10), FuncEnd(0),
+        Prim(lib._CFFI_PRIM_LONG),
+        Prim(lib._CFFI_PRIM_SHORT)]
+    assert parse("int(char[])") == [
+        Prim(lib._CFFI_PRIM_INT),
+        '->', Func(0), Pointer(4), FuncEnd(0),
+        Prim(lib._CFFI_PRIM_CHAR),
+        OpenArray(4)]

From noreply at buildbot.pypy.org  Fri Apr 10 19:16:17 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 10 Apr 2015 19:16:17 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: The non-private part of the accepted
	grant proposal
Message-ID: <20150410171617.931911C0318@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1690:16dc660b4ee7
Date: 2015-04-10 19:16 +0200
http://bitbucket.org/cffi/cffi/changeset/16dc660b4ee7/

Log:	The non-private part of the accepted grant proposal

diff --git a/new/cffi-1.0.rst b/new/cffi-1.0.rst
new file mode 100644
--- /dev/null
+++ b/new/cffi-1.0.rst
@@ -0,0 +1,124 @@
+
+===========================
+Grant Proposal for CFFI 1.0
+===========================
+
+*Accepted by the PSF board on April 4, 2015*
+
+This Grant Proposal is to give a boost towards "CFFI 1.0".  Two main
+issues with the current CFFI need to be solved: the difficulties of
+installation, and the potentially large time taken at import.
+
+1. The difficulties of installation can be seen from outside by looking
+at various workarounds and 3rd-party documentation that have grown into
+existence.  For example, the `setup.py` of projects like cryptography,
+PyNaCl and bcrypt deploys workarounds that are explicitly documented in
+https://caremad.io/2014/11/distributing-a-cffi-project/.
+
+2. The time taken at import is excessive in some cases.  For example,
+importing `pygame-cffi` on a Raspberry Pi ARM board takes on the order
+of 10 to 20 seconds (and this is the "fast" case where the compiler
+doesn't need to be invoked any more).
+
+
+Technical Overview
+------------------
+
+"CFFI" is an existing Python project which complements the ctypes,
+SWIG and Cython approaches to ease writing C Extension Modules for
+Python.  It has several advantages over the previous approaches, which
+are presented at the start of the documentation at
+http://cffi.readthedocs.org/en/latest/ .  It has been very successful
+so far: http://pypi-ranking.info/alltime records almost 7 million
+downloads (for comparison, the #1 of all packages has almost 36
+million downloads).  CFFI works on any Python >= 2.6, including 3.x,
+as well as on PyPy.
+
+One problem is that while getting started with CFFI is very easy, the
+installation process of a package that uses CFFI has got its rough
+edges.  CFFI (at least in its "verify()" mode) is based on calling the
+C compiler to get information about the exact C types, structures,
+argument types to functions, and so on.  The C compiler is invoked
+transparently at run-time, and the results cached.  A
+correctly-installed package using CFFI should cache the results at
+installation time, but it can be difficult to ensure that no more
+run-time compiler invocation is needed; doing so requires following
+some extra guidelines or understanding some internal details.  (The
+problem is particularly acute on Windows where a typical user might
+not have a proper C compiler installed.)
+
+To fix this, we have in mind adding a different CFFI mode (replacing
+"verify()"), while keeping the access to the underlying C library
+unmodified.  In this mode, the code containing the cdef() and verify()
+invocations would be moved to a separate Python source file.  Running
+that Python file would produce a dynamically-linked library.  There
+would be no caching logic involved; you would need to run it
+explicitly during development whenever you made changes to it, to
+re-generate and re-compile the dynamically-linked library.
+
+When distributed, the same file would be run (once) during
+installation.  This can be fully automated in setuptools-based
+setup.py files; alternatively, it can be done in distutils-based
+setup.py files by requiring prior manual installation of CFFI itself.
+
+A major difference with the existing verify() approach would be that
+the ``.so/.dll/.dylib`` file would not be immediately loaded into the
+process; you would load it only from the installed program at
+run-time, and get the ``ffi`` and ``lib`` objects in this way (these
+are the two objects that you use so far to access a C library with
+verify()).
+
+Additionally, this would solve another issue: every import of a large
+CFFI-using package takes a while so far.  This is caused by CFFI
+needing to parse again the C source code given in the cdef() (adding a
+run-time dependency to the ``pycparser`` and ``ply`` packages).  CFFI
+also computes a CRC to know if it can reuse its cache.  In the
+proposed change, all the cdef() code would be pre-parsed and stored in
+the dynamically-linked library, and no CRC would be needed.  This
+would massively reduce the import times.
+
+
+Grant objective
+---------------
+
+The objective is to give a boost towards "CFFI 1.0", which needs to have
+the functionalities described above in order to solve the two main
+issues with the current CFFI: the difficulties of installation, and the
+time taken at import.
+
+Included in the objective: the internal refactorings of CFFI that are
+needed to get it done cleanly.  The goal is to avoid simply adding
+another layer on top of the old unchanged CFFI.
+
+This work may happen eventually in any case, but support from the PSF
+would help make it happen sooner rather than later.
+
+
+Grant size
+----------
+
+2'500 US$ for supporting the development time.  This would cover 2.5
+weeks of full-time work at the part-time cost of 25 US$ per hour.
+
+The estimated work time until the CFFI 1.0 release is a bit larger
+than that (I estimate it at roughly 4 weeks), but 2.5 weeks should
+cover all the basics.  An extended grant size of 4'000 US$ would be
+appreciated but not required ``:-)``
+
+
+Grant beneficiaries
+-------------------
+
+Armin Rigo, main author of CFFI, committing 2.5 weeks of full-time
+work.
+
+
+Grant follow-up
+---------------
+
+I will report on the success of the grant on the CFFI mailing list and
+on the blog I usually post to (the PyPy blog) and mention the PSF as
+providing the grant.  The PSF will receive an email pointing to these
+postings once they are out.  Moreover a full CFFI 1.0 release should
+follow (likely starting with beta versions); the PSF will receive
+another email pointing to it.

From noreply at buildbot.pypy.org  Fri Apr 10 19:19:53 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 10 Apr 2015 19:19:53 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: move the assert next to the line it is
	protecting
Message-ID: <20150410171953.432131C1016@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1691:8236b06c316b
Date: 2015-04-10 19:20 +0200
http://bitbucket.org/cffi/cffi/changeset/8236b06c316b/

Log:	move the assert next to the line it is protecting

diff --git a/new/parse_c_type.c b/new/parse_c_type.c
--- a/new/parse_c_type.c
+++ b/new/parse_c_type.c
@@ -286,7 +286,6 @@
                     }
                     int arg = parse_complete(tok);
                     _cffi_opcode_t oarg;
-                    assert(arg_next - base_index <= arg_total);
                     switch (_CFFI_GETOP(tok->output[arg])) {
                     case _CFFI_OP_ARRAY:
                     case _CFFI_OP_OPEN_ARRAY:
@@ -299,6 +298,7 @@
                         oarg = _CFFI_OP(_CFFI_OP_NOOP, arg);
                         break;
                     }
+                    assert(arg_next - base_index <= arg_total);
                     tok->output[arg_next++] = oarg;
                     if (tok->kind != TOK_COMMA)
                         break;

From noreply at buildbot.pypy.org  Fri Apr 10 19:49:29 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Fri, 10 Apr 2015 19:49:29 +0200 (CEST)
Subject: [pypy-commit] pypy issue2018: fix SingleFrozenPBCR.convert_const()
Message-ID: <20150410174929.797D21C01F3@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: issue2018
Changeset: r76770:516fd6b31b0c
Date: 2015-04-10 18:46 +0100
http://bitbucket.org/pypy/pypy/changeset/516fd6b31b0c/

Log:	fix SingleFrozenPBCR.convert_const()

diff --git a/rpython/rtyper/rpbc.py b/rpython/rtyper/rpbc.py
--- a/rpython/rtyper/rpbc.py
+++ b/rpython/rtyper/rpbc.py
@@ -383,6 +383,9 @@
         assert frozendesc is self.frozendesc
         return object()  # lowleveltype is Void
 
+    def convert_const(self, value):
+        return None
+
     def getstr(self):
         return str(self.frozendesc)
     getstr._annspecialcase_ = 'specialize:memo'

From noreply at buildbot.pypy.org  Sat Apr 11 00:52:33 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Sat, 11 Apr 2015 00:52:33 +0200 (CEST)
Subject: [pypy-commit] pypy issue2018: unxfail passing tests
Message-ID: <20150410225233.E33F61C1EE7@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: issue2018
Changeset: r76771:c06ba587fc61
Date: 2015-04-10 19:46 +0100
http://bitbucket.org/pypy/pypy/changeset/c06ba587fc61/

Log:	unxfail passing tests

diff --git a/rpython/rtyper/lltypesystem/test/test_lltype.py b/rpython/rtyper/lltypesystem/test/test_lltype.py
--- a/rpython/rtyper/lltypesystem/test/test_lltype.py
+++ b/rpython/rtyper/lltypesystem/test/test_lltype.py
@@ -547,10 +547,10 @@
 
 @py.test.mark.parametrize('x', [
     1, sys.maxint, 1.5, 'a', 'abc', u'abc', None, [],
-    py.test.mark.xfail(lambda: None),
+    lambda: None,
     {1.23: 'abc'},
     (1, 'x', [2, 3.],),
-    py.test.mark.xfail(Frozen()),])
+    Frozen(),])
 def test_typeOf_const(x):
     a = RPythonAnnotator()
     bk = a.bookkeeper

From noreply at buildbot.pypy.org  Sat Apr 11 09:26:31 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 11 Apr 2015 09:26:31 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Structs and unions
Message-ID: <20150411072631.056B61C0ACA@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1692:3162ab4fa3b8
Date: 2015-04-11 09:27 +0200
http://bitbucket.org/cffi/cffi/changeset/3162ab4fa3b8/

Log:	Structs and unions

diff --git a/new/parse_c_type.c b/new/parse_c_type.c
--- a/new/parse_c_type.c
+++ b/new/parse_c_type.c
@@ -1,4 +1,5 @@
 #include 
+#include 
 #include 
 #include "parse_c_type.h"
 
@@ -175,10 +176,8 @@
 {
     if (tok->kind != TOK_ERROR) {
         tok->kind = TOK_ERROR;
-        if (tok->info->error_location)
-            *tok->info->error_location = tok->p;
-        if (tok->info->error_message)
-            *tok->info->error_message = msg;
+        tok->info->error_location = tok->p;
+        tok->info->error_message = msg;
     }
     return -1;
 }
@@ -410,6 +409,25 @@
 }
 #endif
 
+static int search_struct_union(struct _cffi_type_context_s *ctx,
+                               const char *search, size_t search_len)
+{
+    int left = 0, right = ctx->num_structs_unions;
+
+    while (left < right) {
+        int middle = (left + right) / 2;
+        const char *src = ctx->structs_unions[middle].name;
+        int diff = strncmp(src, search, search_len);
+        if (diff == 0 && src[search_len] == '\0')
+            return middle;
+        else if (diff >= 0)
+            right = middle;
+        else
+            left = middle + 1;
+    }
+    return -1;
+}
+
 static int parse_complete(token_t *tok)
 {
  qualifiers:
@@ -554,27 +572,20 @@
         case TOK_STRUCT:
         case TOK_UNION:
         {
-            abort();
-#if 0
-            char identifier[1024];
             int kind = tok->kind;
             next_token(tok);
-            if (tok->kind != TOK_IDENTIFIER) {
-                parse_error(tok, "struct or union name expected");
-                return;
-            }
-            if (tok->size >= 1024) {
-                parse_error(tok, "struct or union name too long");
-                return;
-            }
-            memcpy(identifier, tok->p, tok->size);
-            identifier[tok->size] = 0;
-            if (kind == TOK_STRUCT)
-                t1 = tok->cb->get_struct_type(tok->cb, identifier);
-            else
-                t1 = tok->cb->get_union_type(tok->cb, identifier);
+            if (tok->kind != TOK_IDENTIFIER)
+                return parse_error(tok, "struct or union name expected");
+
+            int n = search_struct_union(tok->info->ctx, tok->p, tok->size);
+            if (n < 0)
+                return parse_error(tok, "undefined struct/union name");
+            if (((tok->info->ctx->structs_unions[n].flags & CT_UNION) != 0)
+                ^ (kind == TOK_UNION))
+                return parse_error(tok, "wrong kind of tag: struct vs union");
+
+            t1 = _CFFI_OP(_CFFI_OP_STRUCT_UNION, n);
             break;
-#endif
         }
         default:
             return parse_error(tok, "identifier expected");
diff --git a/new/parse_c_type.h b/new/parse_c_type.h
--- a/new/parse_c_type.h
+++ b/new/parse_c_type.h
@@ -57,6 +57,8 @@
     int num_fields;
     int first_field_index;   // -> _cffi_fields array
 };
+#define CT_UNION            128
+#define CT_IS_OPAQUE        4096
 
 struct _cffi_field_s {
     const char *name;
@@ -94,8 +96,8 @@
     struct _cffi_type_context_s *ctx;
     _cffi_opcode_t *output;
     int output_size;
-    const char **error_location;
-    const char **error_message;
+    const char *error_location;
+    const char *error_message;
 };
 
 int parse_c_type(struct _cffi_parse_info_s *info, const char *input);
diff --git a/new/test_parse_c_type.py b/new/test_parse_c_type.py
--- a/new/test_parse_c_type.py
+++ b/new/test_parse_c_type.py
@@ -1,5 +1,6 @@
 import re
 import os
+import py
 import cffi
 
 r_macro = re.compile(r"#define \w+[(][^\n]*")
@@ -17,9 +18,23 @@
 class ParseError(Exception):
     pass
 
+struct_names = ["bar_s", "foo", "foo_", "foo_s", "foo_s1", "foo_s12"]
+assert struct_names == sorted(struct_names)
+
+ctx = ffi.new("struct _cffi_type_context_s *")
+c_names = [ffi.new("char[]", _n) for _n in struct_names]
+ctx_structs = ffi.new("struct _cffi_struct_union_s[]", len(struct_names))
+for _i in range(len(struct_names)):
+    ctx_structs[_i].name = c_names[_i]
+ctx_structs[3].flags = lib.CT_UNION
+ctx.structs_unions = ctx_structs
+ctx.num_structs_unions = len(struct_names)
+
+
 def parse(input):
     out = ffi.new("_cffi_opcode_t[]", 100)
     info = ffi.new("struct _cffi_parse_info_s *")
+    info.ctx = ctx
     info.output = out
     info.output_size = len(out)
     for j in range(len(out)):
@@ -28,7 +43,7 @@
     res = lib.parse_c_type(info, c_input)
     if res < 0:
         raise ParseError(ffi.string(info.error_message),
-                         ffi.string(info.error_location) - c_input)
+                         info.error_location - c_input)
     assert 0 <= res < len(out)
     result = []
     for j in range(len(out)):
@@ -49,6 +64,11 @@
         return '%d,%d' % (x & 255, x >> 8)
     return '  '.join(map(str_if_int, result))
 
+def parse_error(input, expected_msg, expected_location):
+    e = py.test.raises(ParseError, parse, input)
+    assert e.value.args[0] == expected_msg
+    assert e.value.args[1] == expected_location
+
 def make_getter(name):
     opcode = getattr(lib, '_CFFI_OP_' + name)
     def getter(value):
@@ -62,6 +82,7 @@
 NoOp = make_getter('NOOP')
 Func = make_getter('FUNCTION')
 FuncEnd = make_getter('FUNCTION_END')
+Struct = make_getter('STRUCT_UNION')
 
 
 def test_simple():
@@ -164,3 +185,14 @@
         '->', Func(0), Pointer(4), FuncEnd(0),
         Prim(lib._CFFI_PRIM_CHAR),
         OpenArray(4)]
+
+def test_error():
+    parse_error("short short int", "'short' after another 'short' or 'long'", 6)
+
+def test_struct():
+    for i in range(len(struct_names)):
+        if i == 3:
+            tag = "union"
+        else:
+            tag = "struct"
+        assert parse("%s %s" % (tag, struct_names[i])) == ['->', Struct(i)]

From noreply at buildbot.pypy.org  Sat Apr 11 09:31:28 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 11 Apr 2015 09:31:28 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Copy some tests from creflect
Message-ID: <20150411073128.069B01C0ACA@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1693:3ac82266ea94
Date: 2015-04-11 09:32 +0200
http://bitbucket.org/cffi/cffi/changeset/3ac82266ea94/

Log:	Copy some tests from creflect

diff --git a/new/parse_c_type.c b/new/parse_c_type.c
--- a/new/parse_c_type.c
+++ b/new/parse_c_type.c
@@ -1,6 +1,7 @@
 #include 
 #include 
 #include 
+#include 
 #include "parse_c_type.h"
 
 
@@ -324,10 +325,13 @@
             if (tok->kind != TOK_INTEGER)
                 return parse_error(tok, "expected a positive integer constant");
 
+            errno = 0;
             if (sizeof(length) > sizeof(unsigned long))
                 length = strtoull(tok->p, NULL, 10);
             else
                 length = strtoul(tok->p, NULL, 10);
+            if (errno == ERANGE)
+                return parse_error(tok, "number too large");
             next_token(tok);
 
             write_ds(tok, _CFFI_OP(_CFFI_OP_ARRAY, 0));
diff --git a/new/test_parse_c_type.py b/new/test_parse_c_type.py
--- a/new/test_parse_c_type.py
+++ b/new/test_parse_c_type.py
@@ -188,6 +188,34 @@
 
 def test_error():
     parse_error("short short int", "'short' after another 'short' or 'long'", 6)
+    parse_error("long long long", "'long long long' is too long", 10)
+    parse_error("short long", "'long' after 'short'", 6)
+    parse_error("signed unsigned int", "multiple 'signed' or 'unsigned'", 7)
+    parse_error("unsigned signed int", "multiple 'signed' or 'unsigned'", 9)
+    parse_error("long char", "invalid combination of types", 5)
+    parse_error("short char", "invalid combination of types", 6)
+    parse_error("signed void", "invalid combination of types", 7)
+    parse_error("unsigned struct", "invalid combination of types", 9)
+    #
+    parse_error("", "identifier expected", 0)
+    parse_error("]", "identifier expected", 0)
+    parse_error("*", "identifier expected", 0)
+    parse_error("int ]**", "unexpected symbol", 4)
+    parse_error("char char", "unexpected symbol", 5)
+    parse_error("int(int]", "expected ')'", 7)
+    parse_error("int(*]", "expected ')'", 5)
+    parse_error("int(]", "identifier expected", 4)
+    parse_error("int[?]", "expected a positive integer constant", 4)
+    parse_error("int[24)", "expected ']'", 6)
+    parse_error("struct", "struct or union name expected", 6)
+    parse_error("struct 24", "struct or union name expected", 7)
+    parse_error("int[5](*)", "unexpected symbol", 6)
+    parse_error("int a(*)", "identifier expected", 6)
+    parse_error("int[123456789012345678901234567890]", "number too large", 4)
+
+def test_complexity_limit():
+    parse_error("int" + "[]" * 2500, "internal type complexity limit reached",
+                202)
 
 def test_struct():
     for i in range(len(struct_names)):

From noreply at buildbot.pypy.org  Sat Apr 11 09:39:20 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 11 Apr 2015 09:39:20 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Typenames
Message-ID: <20150411073920.432121C0107@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1694:54d0281c9072
Date: 2015-04-11 09:39 +0200
http://bitbucket.org/cffi/cffi/changeset/54d0281c9072/

Log:	Typenames

diff --git a/new/parse_c_type.c b/new/parse_c_type.c
--- a/new/parse_c_type.c
+++ b/new/parse_c_type.c
@@ -432,6 +432,25 @@
     return -1;
 }
 
+static int search_typename(struct _cffi_type_context_s *ctx,
+                           const char *search, size_t search_len)
+{
+    int left = 0, right = ctx->num_typenames;
+
+    while (left < right) {
+        int middle = (left + right) / 2;
+        const char *src = ctx->typenames[middle].name;
+        int diff = strncmp(src, search, search_len);
+        if (diff == 0 && src[search_len] == '\0')
+            return middle;
+        else if (diff >= 0)
+            right = middle;
+        else
+            left = middle + 1;
+    }
+    return -1;
+}
+
 static int parse_complete(token_t *tok)
 {
  qualifiers:
@@ -557,21 +576,12 @@
             break;
         case TOK_IDENTIFIER:
         {
-            abort();
-#if 0
-            _crx_qual_type qt2;
-            char identifier[1024];
-            if (tok->size >= 1024) {
-                parse_error(tok, "identifier name too long");
-                return;
-            }
-            memcpy(identifier, tok->p, tok->size);
-            identifier[tok->size] = 0;
-            qt2 = tok->cb->get_user_type(tok->cb, identifier);
-            t1 = qt2.type;
-            result->qualifiers |= qt2.qualifiers;
+            int n = search_typename(tok->info->ctx, tok->p, tok->size);
+            if (n < 0)
+                return parse_error(tok, "undefined type name");
+
+            t1 = _CFFI_OP(_CFFI_OP_TYPENAME, n);
             break;
-#endif
         }
         case TOK_STRUCT:
         case TOK_UNION:
diff --git a/new/parse_c_type.h b/new/parse_c_type.h
--- a/new/parse_c_type.h
+++ b/new/parse_c_type.h
@@ -44,7 +44,7 @@
 struct _cffi_constant_s {
     const char *name;
     unsigned long long value;
-    int type_index_or_plain;
+    int type_index;          // -> _cffi_types or _CFFI_PLAIN_*_INT
 };
 #define _CFFI_PLAIN_POSITIVE_INT     (-1)
 #define _CFFI_PLAIN_NONPOSITIVE_INT  (-2)
diff --git a/new/test_parse_c_type.py b/new/test_parse_c_type.py
--- a/new/test_parse_c_type.py
+++ b/new/test_parse_c_type.py
@@ -21,15 +21,25 @@
 struct_names = ["bar_s", "foo", "foo_", "foo_s", "foo_s1", "foo_s12"]
 assert struct_names == sorted(struct_names)
 
+identifier_names = ["id", "id0", "id05", "id05b", "tail"]
+assert identifier_names == sorted(identifier_names)
+
 ctx = ffi.new("struct _cffi_type_context_s *")
-c_names = [ffi.new("char[]", _n) for _n in struct_names]
+c_struct_names = [ffi.new("char[]", _n) for _n in struct_names]
 ctx_structs = ffi.new("struct _cffi_struct_union_s[]", len(struct_names))
 for _i in range(len(struct_names)):
-    ctx_structs[_i].name = c_names[_i]
+    ctx_structs[_i].name = c_struct_names[_i]
 ctx_structs[3].flags = lib.CT_UNION
 ctx.structs_unions = ctx_structs
 ctx.num_structs_unions = len(struct_names)
 
+c_identifier_names = [ffi.new("char[]", _n) for _n in identifier_names]
+ctx_identifiers = ffi.new("struct _cffi_typename_s[]", len(identifier_names))
+for _i in range(len(identifier_names)):
+    ctx_identifiers[_i].name = c_identifier_names[_i]
+ctx.typenames = ctx_identifiers
+ctx.num_typenames = len(identifier_names)
+
 
 def parse(input):
     out = ffi.new("_cffi_opcode_t[]", 100)
@@ -83,6 +93,7 @@
 Func = make_getter('FUNCTION')
 FuncEnd = make_getter('FUNCTION_END')
 Struct = make_getter('STRUCT_UNION')
+Typename = make_getter('TYPENAME')
 
 
 def test_simple():
@@ -224,3 +235,11 @@
         else:
             tag = "struct"
         assert parse("%s %s" % (tag, struct_names[i])) == ['->', Struct(i)]
+        assert parse("%s %s*" % (tag, struct_names[i])) == [Struct(i),
+                                                            '->', Pointer(0)]
+
+def test_identifier():
+    for i in range(len(identifier_names)):
+        assert parse("%s" % (identifier_names[i])) == ['->', Typename(i)]
+        assert parse("%s*" % (identifier_names[i])) == [Typename(i),
+                                                        '->', Pointer(0)]

From noreply at buildbot.pypy.org  Sat Apr 11 11:30:40 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 11 Apr 2015 11:30:40 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Add the abstract for STM for
	EuroPython 2015
Message-ID: <20150411093040.DF1C81C0845@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: extradoc
Changeset: r5525:9fd26448fe0f
Date: 2015-04-11 11:31 +0200
http://bitbucket.org/pypy/extradoc/changeset/9fd26448fe0f/

Log:	Add the abstract for STM for EuroPython 2015

diff --git a/talk/ep2015/stm-abstract.rst b/talk/ep2015/stm-abstract.rst
new file mode 100644
--- /dev/null
+++ b/talk/ep2015/stm-abstract.rst
@@ -0,0 +1,21 @@
+=========================
+The GIL is dead: PyPy-STM
+=========================
+
+Abstract
+--------
+
+Take a big, non-multithreaded program, and run in on multiple cores!
+
+PyPy, the Python implementation written in Python, experimentally
+supports Software Transactional Memory (STM).  It runs without the
+Global Interpreter Lock (GIL).
+
+The strength of STM is not only to remove the GIL, but to also enable
+a novel use of multithreading, inheritently safe, and more useful in
+the general case than other approaches like OpenMP.  The main news
+from last year's presentation is that there is now a way to get
+reports about the "STM conflicts", which is essential to go past toy
+examples.  With it, you can incrementally remove conflicts from large
+code bases until you see a benefit from PyPy-STM.  The goal of the
+talk is to give several concrete examples of doing that.

From noreply at buildbot.pypy.org  Sat Apr 11 11:49:42 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 11 Apr 2015 11:49:42 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix: there are roughly 80'000 symbols
 '__gcmap_*' in an asmgcc pypy, and
Message-ID: <20150411094942.C38841C0845@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76772:2769d419e952
Date: 2015-04-11 10:39 +0200
http://bitbucket.org/pypy/pypy/changeset/2769d419e952/

Log:	Fix: there are roughly 80'000 symbols '__gcmap_*' in an asmgcc pypy,
	and they were not declared '.hidden', which means that they were
	left in the binary. Moreover, they were relocated at start-up,
	which took an appreciable fraction of second.

diff --git a/rpython/translator/c/gcc/trackgcroot.py b/rpython/translator/c/gcc/trackgcroot.py
--- a/rpython/translator/c/gcc/trackgcroot.py
+++ b/rpython/translator/c/gcc/trackgcroot.py
@@ -423,6 +423,7 @@
                 # the original value for gcmaptable.s.  That's a hack.
                 self.lines.insert(call.lineno+1, '%s=.+%d\n' % (label,
                                                                 self.OFFSET_LABELS))
+                self.lines.insert(call.lineno+1, '\t.hidden\t%s\n' % (label,))
                 self.lines.insert(call.lineno+1, '\t.globl\t%s\n' % (label,))
         call.global_label = label
 

From noreply at buildbot.pypy.org  Sat Apr 11 11:49:44 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 11 Apr 2015 11:49:44 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix test
Message-ID: <20150411094944.015641C0845@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76773:60de3e60e9d5
Date: 2015-04-11 10:42 +0200
http://bitbucket.org/pypy/pypy/changeset/60de3e60e9d5/

Log:	fix test

diff --git a/rpython/translator/c/gcc/test/test_trackgcroot.py b/rpython/translator/c/gcc/test/test_trackgcroot.py
--- a/rpython/translator/c/gcc/test/test_trackgcroot.py
+++ b/rpython/translator/c/gcc/test/test_trackgcroot.py
@@ -159,7 +159,8 @@
                 expectedlines.insert(i-2, 'PUBLIC\t%s\n' % (label,))
                 expectedlines.insert(i-1, '%s::\n' % (label,))
             else:
-                expectedlines.insert(i-2, '\t.globl\t%s\n' % (label,))
+                expectedlines.insert(i-3, '\t.globl\t%s\n' % (label,))
+                expectedlines.insert(i-2, '\t.hidden\t%s\n' % (label,))
                 expectedlines.insert(i-1, '%s=.+%d\n' % (label,
                                                          tracker.OFFSET_LABELS))
         if format == 'msvc' and r_gcroot_constant.match(line):

From noreply at buildbot.pypy.org  Sat Apr 11 18:25:16 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 11 Apr 2015 18:25:16 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Realizing C types from parsed
 _cffi_opcode arrays: write the basics, 
Message-ID: <20150411162516.F2D041C08C1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1695:d5b493dd8a62
Date: 2015-04-11 18:25 +0200
http://bitbucket.org/cffi/cffi/changeset/d5b493dd8a62/

Log:	Realizing C types from parsed _cffi_opcode arrays: write the basics,
	needs to be integrated with _cffi_backend.c

diff --git a/new/parse_c_type.c b/new/parse_c_type.c
--- a/new/parse_c_type.c
+++ b/new/parse_c_type.c
@@ -349,70 +349,6 @@
     return _CFFI_GETARG(result);
 }
 
-#if 0
-static void fetch_delay_slots(token_t *tok, _crx_qual_type *result,
-                              intptr_t *delay_slot)
-{
-    if (tok->kind == TOK_ERROR)
-        return;
-    tok->delay_slots = delay_slot;
-    while (1) {
-        intptr_t tok_kind = *delay_slot++;
-        if (tok_kind <= 0) {
-            delay_slot = tok->all_delay_slots + (-tok_kind);
-            continue;
-        }
-        switch (tok_kind) {
-        case TOK_END:
-            return;    /* done */
-        case TOK_STAR:
-            result->type = tok->cb->get_pointer_type(tok->cb, result->type,
-                                                     result->qualifiers);
-            result->qualifiers = 0;
-            break;
-        case TOK_CONST:
-            result->qualifiers |= _CRX_CONST;
-            break;
-        case TOK_VOLATILE:
-            result->qualifiers |= _CRX_VOLATILE;
-            break;
-        case TOK_OPEN_BRACKET:   /* array */
-            {
-                uintptr_t length = (uintptr_t)*delay_slot++;
-                if (length != (uintptr_t)-1)
-                    result->type = tok->cb->get_array_type(
-                        tok->cb, result->type, length);
-                else
-                    result->type = tok->cb->get_incomplete_array_type(
-                        tok->cb, result->type);
-                /* result->qualifiers remains unmodified */
-                break;
-            }
-        case TOK_OPEN_PAREN:   /* function */
-        case TOK_DOTDOTDOT:    /* function ending with a '...' */
-            {
-                intptr_t nbargs = *delay_slot++;
-                _crx_type_t *t1;
-                _crx_qual_type *argtypes = (_crx_qual_type *)delay_slot;
-                delay_slot += 2 * nbargs;
-                if (tok_kind == TOK_DOTDOTDOT)
-                    t1 = tok->cb->get_ellipsis_function_type(tok->cb,
-                                                             result->type,
-                                                             argtypes, nbargs);
-                else
-                    t1 = tok->cb->get_function_type(tok->cb, result->type,
-                                                    argtypes, nbargs, NULL);
-                result->type = t1;
-                result->qualifiers = 0; /* drop qualifiers on the return type */
-                break;
-            }
-        default:
-            assert(!"missing delay slot case");
-        }
-    }
-}
-#endif
-
 static int search_struct_union(struct _cffi_type_context_s *ctx,
                                const char *search, size_t search_len)
 {
diff --git a/new/parse_c_type.h b/new/parse_c_type.h
--- a/new/parse_c_type.h
+++ b/new/parse_c_type.h
@@ -1,3 +1,4 @@
+#include 
 
 
 typedef void *_cffi_opcode_t;
@@ -33,6 +34,7 @@
 #define _CFFI_PRIM_FLOAT        13
 #define _CFFI_PRIM_DOUBLE       14
 #define _CFFI_PRIM_LONGDOUBLE   15
+#define _CFFI__NUM_PRIM         16
 
 
 struct _cffi_global_s {
@@ -79,6 +81,7 @@
 };
 
 struct _cffi_type_context_s {
+    _cffi_opcode_t *types;
     const struct _cffi_global_s *globals;
     const struct _cffi_constant_s *constants;
     const struct _cffi_struct_union_s *structs_unions;
diff --git a/new/realize_c_type.c b/new/realize_c_type.c
new file mode 100644
--- /dev/null
+++ b/new/realize_c_type.c
@@ -0,0 +1,97 @@
+#include 
+#include "parse_c_type.h"
+
+
+static PyObject *all_primitives[_CFFI__NUM_PRIM];
+
+
+PyObject *build_primitive_type(int num)
+{
+    PyObject *x;
+
+    switch (num) {
+
+    case _CFFI_PRIM_VOID:
+        x = PyString_FromString("VOID");
+        break;
+
+    case _CFFI_PRIM_INT:
+        x = PyString_FromString("INT");
+        break;
+
+    default:
+        PyErr_Format(PyExc_NotImplementedError, "prim=%d", num);
+        return NULL;
+    }
+
+    all_primitives[num] = x;
+    return x;
+}
+
+
+PyObject *realize_c_type(struct _cffi_type_context_s *ctx,
+                         _cffi_opcode_t opcodes[], int index)
+{
+    PyObject *x, *y;
+    _cffi_opcode_t op = opcodes[index];
+
+    switch (_CFFI_GETOP(op)) {
+
+    case _CFFI_OP_PRIMITIVE:
+        x = all_primitives[_CFFI_GETARG(op)];
+        if (x == NULL)
+            x = build_primitive_type(_CFFI_GETARG(op));
+        Py_XINCREF(x);
+        return x;
+
+    case _CFFI_OP_POINTER:
+        y = realize_c_type(ctx, opcodes, _CFFI_GETARG(op));
+        if (y == NULL)
+            return NULL;
+        x = Py_BuildValue("sO", "pointer", y);
+        Py_DECREF(y);
+        return x;
+
+    default:
+        PyErr_Format(PyExc_NotImplementedError, "op=%d", (int)_CFFI_GETOP(op));
+        return NULL;
+    }
+}
+
+
+struct _cffi_type_context_s global_ctx = {
+};
+
+
+static PyObject *b_test(PyObject *self, PyObject *args)
+{
+    char *s;
+    if (!PyArg_ParseTuple(args, "s", &s))
+        return NULL;
+
+    _cffi_opcode_t opcodes[100];
+    struct _cffi_parse_info_s parse_info = {
+        .ctx = &global_ctx,
+        .output = opcodes,
+        .output_size = 100,
+    };
+    int res = parse_c_type(&parse_info, s);
+    if (res < 0) {
+        PyErr_SetString(PyExc_ValueError, parse_info.error_message);
+        return NULL;
+    }
+
+    return realize_c_type(&global_ctx, opcodes, res);
+}
+
+static PyMethodDef MyMethods[] = {
+    {"test",   b_test,  METH_VARARGS},
+    {NULL,     NULL}    /* Sentinel */
+};
+
+PyMODINIT_FUNC
+initrealize_c_type(void)
+{
+    PyObject *m = Py_InitModule("realize_c_type", MyMethods);
+    (void)m;
+}
diff --git a/new/setup.py b/new/setup.py
new file mode 100644
--- /dev/null
+++ b/new/setup.py
@@ -0,0 +1,6 @@
+from distutils.core import setup
+from distutils.extension import Extension
+setup(name='realize_c_type',
+      ext_modules=[Extension(name='realize_c_type',
+                             sources=['realize_c_type.c',
+                                      'parse_c_type.c'])])
diff --git a/new/test_parse_c_type.py b/new/test_parse_c_type.py
--- a/new/test_parse_c_type.py
+++ b/new/test_parse_c_type.py
@@ -3,7 +3,7 @@
 import py
 import cffi
 
-r_macro = re.compile(r"#define \w+[(][^\n]*")
+r_macro = re.compile(r"#define \w+[(][^\n]*|#include [^\n]*")
 r_define = re.compile(r"(#define \w+) [^\n]*")
 header = open('parse_c_type.h').read()
 header = r_macro.sub(r"", header)
diff --git a/new/test_realize_c_type.py b/new/test_realize_c_type.py
new file mode 100644
--- /dev/null
+++ b/new/test_realize_c_type.py
@@ -0,0 +1,13 @@
+import os
+
+def setup_module():
+    os.system("python setup.py build_ext -i")
+    global realize_c_type
+    import realize_c_type
+
+
+def test_void():
+    assert realize_c_type.test("void") == "VOID"
+
+def test_int_star():
+    assert realize_c_type.test("int *") == ("pointer", "INT")

From noreply at buildbot.pypy.org  Sat Apr 11 21:17:25 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sat, 11 Apr 2015 21:17:25 +0200 (CEST)
Subject: [pypy-commit] pypy object-dtype2: differentiate between _bool on
 ComplexType and ObjectType
Message-ID: <20150411191725.48C6A1C0134@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: object-dtype2
Changeset: r76774:2f804b4e682f
Date: 2015-04-11 22:18 +0300
http://bitbucket.org/pypy/pypy/changeset/2f804b4e682f/

Log:	differentiate between _bool on ComplexType and ObjectType

diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -149,8 +149,7 @@
         return self.box(array[0])
 
     def unbox(self, box):
-        if isinstance(box, ObjectType.BoxType):
-            return box.w_obj
+        assert isinstance(box, self.BoxType)
         return box.value
 
     def coerce(self, space, dtype, w_item):
@@ -1255,25 +1254,25 @@
     def ge(self, v1, v2):
         return self._lt(v2, v1) or self._eq(v2, v1)
 
-    def _bool(self, v):
+    def _cbool(self, v):
         return bool(v[0]) or bool(v[1])
 
     @raw_binary_op
     def logical_and(self, v1, v2):
-        return self._bool(v1) and self._bool(v2)
+        return self._cbool(v1) and self._cbool(v2)
 
     @raw_binary_op
     def logical_or(self, v1, v2):
-        return self._bool(v1) or self._bool(v2)
+        return self._cbool(v1) or self._cbool(v2)
 
     @raw_unary_op
     def logical_not(self, v):
-        return not self._bool(v)
+        return not self._cbool(v)
 
     @raw_binary_op
     def logical_xor(self, v1, v2):
-        a = self._bool(v1)
-        b = self._bool(v2)
+        a = self._cbool(v1)
+        b = self._cbool(v2)
         return (not b and a) or (not a and b)
 
     def min(self, v1, v2):
@@ -1736,33 +1735,33 @@
 
     @simple_unary_op
     def bool(self,v):
-        return self._bool(v)
+        return self._obool(v)
 
-    def _bool(self, v):
+    def _obool(self, v):
         if self.space.is_true(v):
             return True 
         return False
 
     @simple_binary_op
     def logical_and(self, v1, v2):
-        if self._bool(v1):
+        if self._obool(v1):
             return v2
         return v1
 
     @simple_binary_op
     def logical_or(self, v1, v2):
-        if self._bool(v1):
+        if self._obool(v1):
             return v1
         return v2
 
     @simple_unary_op
     def logical_not(self, v):
-        return not self._bool(v)
+        return not self._obool(v)
 
     @simple_binary_op
     def logical_xor(self, v1, v2):
-        a = self._bool(v1)
-        b = self._bool(v2)
+        a = self._obool(v1)
+        b = self._obool(v2)
         return (not b and a) or (not a and b)
 
     @simple_binary_op

From noreply at buildbot.pypy.org  Sat Apr 11 22:07:59 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 11 Apr 2015 22:07:59 +0200 (CEST)
Subject: [pypy-commit] pypy default: Issue #2023: cpyext: in the 'Concrete
	Object Layer' API, 
Message-ID: <20150411200759.CF5B81C0845@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: 
Changeset: r76775:5580e5171f5f
Date: 2015-04-11 22:05 +0200
http://bitbucket.org/pypy/pypy/changeset/5580e5171f5f/

Log:	Issue #2023: cpyext: in the 'Concrete Object Layer' API, don't call
	methods on the object (which can be overriden), but directly on the
	concrete base type.

	PyDict_Keys(obj) should call dict.keys(obj) and not obj.keys().

diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -91,39 +91,39 @@
 @cpython_api([PyObject], lltype.Void)
 def PyDict_Clear(space, w_obj):
     """Empty an existing dictionary of all key-value pairs."""
-    space.call_method(w_obj, "clear")
+    space.call_method(space.w_dict, "clear", w_obj)
 
 @cpython_api([PyObject], PyObject)
 def PyDict_Copy(space, w_obj):
     """Return a new dictionary that contains the same key-value pairs as p.
     """
-    return space.call_method(w_obj, "copy")
+    return space.call_method(space.w_dict, "copy", w_obj)
 
 @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1)
 def PyDict_Update(space, w_obj, w_other):
     """This is the same as PyDict_Merge(a, b, 1) in C, or a.update(b) in
     Python.  Return 0 on success or -1 if an exception was raised.
     """
-    space.call_method(w_obj, "update", w_other)
+    space.call_method(space.w_dict, "update", w_obj, w_other)
     return 0
 
 @cpython_api([PyObject], PyObject)
 def PyDict_Keys(space, w_obj):
     """Return a PyListObject containing all the keys from the dictionary,
     as in the dictionary method dict.keys()."""
-    return space.call_method(w_obj, "keys")
+    return space.call_method(space.w_dict, "keys", w_obj)
 
 @cpython_api([PyObject], PyObject)
 def PyDict_Values(space, w_obj):
     """Return a PyListObject containing all the values from the
     dictionary p, as in the dictionary method dict.values()."""
-    return space.call_method(w_obj, "values")
+    return space.call_method(space.w_dict, "values", w_obj)
 
 @cpython_api([PyObject], PyObject)
 def PyDict_Items(space, w_obj):
     """Return a PyListObject containing all the items from the
     dictionary, as in the dictionary method dict.items()."""
-    return space.call_method(w_obj, "items")
+    return space.call_method(space.w_dict, "items", w_obj)
 
 @cpython_api([PyObject, Py_ssize_tP, PyObjectP, PyObjectP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDict_Next(space, w_dict, ppos, pkey, pvalue):
@@ -175,7 +175,7 @@
     # not complete.
 
     try:
-        w_iter = space.call_method(w_dict, "iteritems")
+        w_iter = space.call_method(space.w_dict, "iteritems", w_dict)
         pos = ppos[0]
         while pos:
             space.call_method(w_iter, "next")
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -65,7 +65,7 @@
     """Insert the item item into list list in front of index index.  Return
     0 if successful; return -1 and set an exception if unsuccessful.
     Analogous to list.insert(index, item)."""
-    space.call_method(w_list, "insert", space.wrap(index), w_item)
+    space.call_method(space.w_list, "insert", w_list, space.wrap(index), w_item)
     return 0
 
 @cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL)
@@ -98,7 +98,7 @@
     failure.  This is equivalent to list.sort()."""
     if not isinstance(w_list, W_ListObject):
         PyErr_BadInternalCall(space)
-    space.call_method(w_list, "sort")
+    space.call_method(space.w_list, "sort", w_list)
     return 0
 
 @cpython_api([PyObject], rffi.INT_real, error=-1)
@@ -107,7 +107,7 @@
     failure.  This is the equivalent of list.reverse()."""
     if not isinstance(w_list, W_ListObject):
         PyErr_BadInternalCall(space)
-    space.call_method(w_list, "reverse")
+    space.call_method(space.w_list, "reverse", w_list)
     return 0
 
 @cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject)
diff --git a/pypy/module/cpyext/setobject.py b/pypy/module/cpyext/setobject.py
--- a/pypy/module/cpyext/setobject.py
+++ b/pypy/module/cpyext/setobject.py
@@ -36,7 +36,7 @@
     values of brand new frozensets before they are exposed to other code."""
     if not PySet_Check(space, w_s):
         PyErr_BadInternalCall(space)
-    space.call_method(w_s, 'add', w_obj)
+    space.call_method(space.w_set, 'add', w_s, w_obj)
     return 0
 
 @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1)
@@ -49,7 +49,7 @@
     instance of set or its subtype."""
     if not PySet_Check(space, w_s):
         PyErr_BadInternalCall(space)
-    space.call_method(w_s, 'discard', w_obj)
+    space.call_method(space.w_set, 'discard', w_s, w_obj)
     return 0
 
 
@@ -59,12 +59,12 @@
     object from the set.  Return NULL on failure.  Raise KeyError if the
     set is empty. Raise a SystemError if set is an not an instance of
     set or its subtype."""
-    return space.call_method(w_set, "pop")
+    return space.call_method(space.w_set, "pop", w_set)
 
 @cpython_api([PyObject], rffi.INT_real, error=-1)
 def PySet_Clear(space, w_set):
     """Empty an existing set of all elements."""
-    space.call_method(w_set, 'clear')
+    space.call_method(space.w_set, 'clear', w_set)
     return 0
 
 @cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL)

From noreply at buildbot.pypy.org  Sun Apr 12 14:38:02 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Sun, 12 Apr 2015 14:38:02 +0200 (CEST)
Subject: [pypy-commit] pypy issue2018: Close branch issue2018
Message-ID: <20150412123802.A2B841C071E@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: issue2018
Changeset: r76776:7cffb392b046
Date: 2015-04-12 13:38 +0100
http://bitbucket.org/pypy/pypy/changeset/7cffb392b046/

Log:	Close branch issue2018


From noreply at buildbot.pypy.org  Sun Apr 12 14:38:08 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Sun, 12 Apr 2015 14:38:08 +0200 (CEST)
Subject: [pypy-commit] pypy default: Merged issue2018 into default
Message-ID: <20150412123808.988ED1C071E@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: 
Changeset: r76777:a8e0fff6383f
Date: 2015-04-12 13:38 +0100
http://bitbucket.org/pypy/pypy/changeset/a8e0fff6383f/

Log:	Merged issue2018 into default

	Fixes #2018

diff --git a/rpython/rtyper/lltypesystem/test/test_lltype.py b/rpython/rtyper/lltypesystem/test/test_lltype.py
--- a/rpython/rtyper/lltypesystem/test/test_lltype.py
+++ b/rpython/rtyper/lltypesystem/test/test_lltype.py
@@ -547,10 +547,10 @@
 
 @py.test.mark.parametrize('x', [
     1, sys.maxint, 1.5, 'a', 'abc', u'abc', None, [],
-    py.test.mark.xfail(lambda: None),
+    lambda: None,
     {1.23: 'abc'},
     (1, 'x', [2, 3.],),
-    py.test.mark.xfail(Frozen()),])
+    Frozen(),])
 def test_typeOf_const(x):
     a = RPythonAnnotator()
     bk = a.bookkeeper
diff --git a/rpython/rtyper/rpbc.py b/rpython/rtyper/rpbc.py
--- a/rpython/rtyper/rpbc.py
+++ b/rpython/rtyper/rpbc.py
@@ -383,6 +383,9 @@
         assert frozendesc is self.frozendesc
         return object()  # lowleveltype is Void
 
+    def convert_const(self, value):
+        return None
+
     def getstr(self):
         return str(self.frozendesc)
     getstr._annspecialcase_ = 'specialize:memo'

From noreply at buildbot.pypy.org  Sun Apr 12 15:51:52 2015
From: noreply at buildbot.pypy.org (Julian Berman)
Date: Sun, 12 Apr 2015 15:51:52 +0200 (CEST)
Subject: [pypy-commit] pypy install-rpython: Install scripts along with
	rpython.
Message-ID: <20150412135152.928A61C00FC@cobra.cs.uni-duesseldorf.de>

Author: Julian Berman 
Branch: install-rpython
Changeset: r76778:bf6fdc9602b4
Date: 2015-04-12 09:14 -0400
http://bitbucket.org/pypy/pypy/changeset/bf6fdc9602b4/

Log:	Install scripts along with rpython.

diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -4,10 +4,10 @@
 from setuptools import setup, find_packages
 # To use a consistent encoding
 from codecs import open
-from os import path
+import os
 
-here = path.abspath(path.dirname(__file__))
-with open(path.join(here, 'README-rpython.rst'), encoding='utf-8') as f:
+here = os.path.abspath(os.path.dirname(__file__))
+with open(os.path.join(here, 'README-rpython.rst'), encoding='utf-8') as f:
     long_description = f.read()
 
 long_description += """
@@ -20,7 +20,7 @@
 
 setup(
     name='rpython',
-    version='0.1.2',
+    version='0.1.3',
     description='RPython',
     long_description=long_description,
 
@@ -43,4 +43,8 @@
     packages=find_packages(),  # MANIFEST.in filters out all the pypy stuff
     package_data={'rpython': ['translator/c/src/*.c', 'translator/c/src/*.h']},
     install_requires=['pytest'],
+    scripts=[
+        os.path.join("rpython", "bin", script)
+        for script in os.listdir("rpython/bin")
+    ],
 )

From noreply at buildbot.pypy.org  Sun Apr 12 16:33:01 2015
From: noreply at buildbot.pypy.org (Julian Berman)
Date: Sun, 12 Apr 2015 16:33:01 +0200 (CEST)
Subject: [pypy-commit] pypy install-rpython: Distutils doesn't know how to
 handle /usr/bin/env pypy for shebang rewrites.
Message-ID: <20150412143301.3AA591C0976@cobra.cs.uni-duesseldorf.de>

Author: Julian Berman 
Branch: install-rpython
Changeset: r76779:34848732c2b4
Date: 2015-04-12 10:27 -0400
http://bitbucket.org/pypy/pypy/changeset/34848732c2b4/

Log:	Distutils doesn't know how to handle /usr/bin/env pypy for shebang
	rewrites.

diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -43,8 +43,9 @@
     packages=find_packages(),  # MANIFEST.in filters out all the pypy stuff
     package_data={'rpython': ['translator/c/src/*.c', 'translator/c/src/*.h']},
     install_requires=['pytest'],
-    scripts=[
-        os.path.join("rpython", "bin", script)
-        for script in os.listdir("rpython/bin")
-    ],
+    entry_points={
+        "console_scripts" : [
+            "rpython = rpython.translator.goal.translate:main",
+        ],
+    },
 )

From noreply at buildbot.pypy.org  Sun Apr 12 17:36:43 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Sun, 12 Apr 2015 17:36:43 +0200 (CEST)
Subject: [pypy-commit] pypy install-rpython: Point 'rpython' to something
 that will give a useful message when run without arguments
Message-ID: <20150412153643.DB73C1C0698@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: install-rpython
Changeset: r76780:689f40c7cd51
Date: 2015-04-12 16:36 +0100
http://bitbucket.org/pypy/pypy/changeset/689f40c7cd51/

Log:	Point 'rpython' to something that will give a useful message when
	run without arguments

diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -45,7 +45,7 @@
     install_requires=['pytest'],
     entry_points={
         "console_scripts" : [
-            "rpython = rpython.translator.goal.translate:main",
+            "rpython = rpython.__main__:main",
         ],
     },
 )

From noreply at buildbot.pypy.org  Sun Apr 12 18:00:00 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Sun, 12 Apr 2015 18:00:00 +0200 (CEST)
Subject: [pypy-commit] pypy install-rpython: Add import guard so that entry
 point calls main() only once
Message-ID: <20150412160000.BB5CE1C04A4@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: install-rpython
Changeset: r76781:995790cd228e
Date: 2015-04-12 16:59 +0100
http://bitbucket.org/pypy/pypy/changeset/995790cd228e/

Log:	Add import guard so that entry point calls main() only once

diff --git a/rpython/__main__.py b/rpython/__main__.py
--- a/rpython/__main__.py
+++ b/rpython/__main__.py
@@ -13,4 +13,6 @@
     sys.exit(1)
 
 from rpython.translator.goal.translate import main
-main()
+
+if __name__ == '__main__':
+    main()
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -20,7 +20,7 @@
 
 setup(
     name='rpython',
-    version='0.1.3',
+    version='0.1.4',
     description='RPython',
     long_description=long_description,
 

From noreply at buildbot.pypy.org  Sun Apr 12 21:31:24 2015
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 12 Apr 2015 21:31:24 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: Add the Guido Seal of Approval
Message-ID: <20150412193124.0CB4A1C06FD@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: extradoc
Changeset: r594:3cf830114ccc
Date: 2015-04-12 15:31 -0400
http://bitbucket.org/pypy/pypy.org/changeset/3cf830114ccc/

Log:	Add the Guido Seal of Approval

diff --git a/index.html b/index.html
--- a/index.html
+++ b/index.html
@@ -72,19 +72,27 @@
 PyPy is a fast, compliant alternative implementation of the Python
 language (2.7.8 and 3.2.5). It has several advantages and distinct features:
 
-
-Speed: thanks to its Just-in-Time compiler, Python programs
-often run faster on PyPy.  (What is a JIT compiler?)
-Memory usage: memory-hungry Python programs (several hundreds of
-MBs or more) might end up taking less space than they do in CPython.
-Compatibility: PyPy is highly compatible with existing python code.
+
+Speed: thanks to its Just-in-Time compiler, Python programs
+often run faster on PyPy.  (What is a JIT compiler?)
+“If you want your code to run faster, you should probably just use PyPy”
+— Guido van Rossum (creator of Python)
+
+Memory usage: memory-hungry Python programs (several hundreds of
+MBs or more) might end up taking less space than they do in CPython.
+
+Compatibility: PyPy is highly compatible with existing python code.
 It supports cffi and can run popular python libraries like twisted
-and django.
-Sandboxing: PyPy provides the ability to run untrusted code in a
-fully secure way.
-Stackless: PyPy comes by default with support for stackless mode,
-providing micro-threads for massive concurrency.
-As well as other features.
+and django.
+
+Sandboxing: PyPy provides the ability to run untrusted code in a
+fully secure way.
+
+Stackless: PyPy comes by default with support for stackless mode,
+providing micro-threads for massive concurrency.
+
+As well as other features.
+
 
 
 Download and try out PyPy
diff --git a/source/index.txt b/source/index.txt
--- a/source/index.txt
+++ b/source/index.txt
@@ -9,6 +9,9 @@
   * **Speed:** thanks to its Just-in-Time compiler, Python programs
     often run `faster`_ on PyPy.  `(What is a JIT compiler?)`_
 
+    "If you want your code to run faster, you should probably just use PyPy."
+    -- Guido van Rossum (creator of Python)
+
   * **Memory usage:** memory-hungry Python programs (several hundreds of
     MBs or more) might end up taking `less space`_ than they do in CPython.
 

From noreply at buildbot.pypy.org  Sun Apr 12 23:10:11 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 12 Apr 2015 23:10:11 +0200 (CEST)
Subject: [pypy-commit] pypy object-dtype2: help translation
Message-ID: <20150412211011.41DB91C0698@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: object-dtype2
Changeset: r76782:3b29ab05eaea
Date: 2015-04-12 20:38 +0300
http://bitbucket.org/pypy/pypy/changeset/3b29ab05eaea/

Log:	help translation

diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -112,12 +112,13 @@
     return dispatcher
 
 class BaseType(object):
-    _immutable_fields_ = ['native', 'space']
+    _immutable_fields_ = ['native', 'space', 'T']
 
     def __init__(self, space, native=True):
         assert isinstance(space, ObjSpace)
         self.native = native
         self.space = space
+        self.T = self.T
 
     def __repr__(self):
         return self.__class__.__name__

From noreply at buildbot.pypy.org  Sun Apr 12 23:10:12 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 12 Apr 2015 23:10:12 +0200 (CEST)
Subject: [pypy-commit] pypy object-dtype2: revert part of f8e674fa7afc,
 explicitly create comp ufuncs
Message-ID: <20150412211012.8EF5A1C0698@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: object-dtype2
Changeset: r76783:c6d28d3e2f8a
Date: 2015-04-13 00:11 +0300
http://bitbucket.org/pypy/pypy/changeset/c6d28d3e2f8a/

Log:	revert part of f8e674fa7afc, explicitly create comp ufuncs

diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -428,25 +428,29 @@
     def default_fromstring(self, space):
         return self.box(0)
 
-    @simple_binary_op
-    def div(self, v1, v2):
+    @specialize.argtype(1, 2)
+    def div(self, b1, b2):
+        v1 = self.for_computation(self.unbox(b1))
+        v2 = self.for_computation(self.unbox(b2))
         if v2 == 0:
-            return 0
+            return self.box(0)
         if (self.T is rffi.SIGNEDCHAR or self.T is rffi.SHORT or self.T is rffi.INT or
                 self.T is rffi.LONG or self.T is rffi.LONGLONG):
             if v2 == -1 and v1 == self.for_computation(most_neg_value_of(self.T)):
-                return 0
-        return v1 / v2
+                return self.box(0)
+        return self.box(v1 / v2)
 
-    @simple_binary_op
-    def floordiv(self, v1, v2):
+    @specialize.argtype(1, 2)
+    def floordiv(self, b1, b2):
+        v1 = self.for_computation(self.unbox(b1))
+        v2 = self.for_computation(self.unbox(b2))
         if v2 == 0:
-            return 0
+            return self.box(0)
         if (self.T is rffi.SIGNEDCHAR or self.T is rffi.SHORT or self.T is rffi.INT or
                 self.T is rffi.LONG or self.T is rffi.LONGLONG):
             if v2 == -1 and v1 == self.for_computation(most_neg_value_of(self.T)):
-                return 0
-        return v1 // v2
+                return self.box(0)
+        return self.box(v1 / v2)
 
     @simple_binary_op
     def mod(self, v1, v2):
@@ -482,7 +486,7 @@
         elif v < 0:
             return -1
         else:
-            #assert v == 0
+            assert v == 0
             return 0
 
     @raw_unary_op
@@ -1633,7 +1637,7 @@
 
 _all_objs_for_tests = [] # for tests
 
-class ObjectType(BaseType):
+class ObjectType(Primitive, BaseType):
     T = lltype.Signed
     BoxType = boxes.W_ObjectBox
 
@@ -1687,6 +1691,8 @@
     def box(self, w_obj):
         if isinstance(w_obj, W_Root):
             pass
+        elif isinstance(w_obj, bool):
+            w_obj = self.space.newbool(w_obj)
         elif isinstance(w_obj, int):
             w_obj = self.space.newint(w_obj)
         elif isinstance(w_obj, lltype.Number):
@@ -1809,6 +1815,26 @@
     def square(self, v):
         return self.space.mul(v, v)
 
+    @simple_binary_op
+    def le(self, v1, v2):
+        return self.space.le(v1, v2)
+
+    @simple_binary_op
+    def ge(self, v1, v2):
+        return self.space.ge(v1, v2)
+
+    @simple_binary_op
+    def lt(self, v1, v2):
+        return self.space.lt(v1, v2)
+
+    @simple_binary_op
+    def gt(self, v1, v2):
+        return self.space.gt(v1, v2)
+
+    @simple_binary_op
+    def ne(self, v1, v2):
+        return self.space.ne(v1, v2)
+
 def add_attributeerr_op(cls, op):
     def func(self, *args):
         raise oefmt(self.space.w_AttributeError, 
@@ -1847,13 +1873,6 @@
     func.__name__ = 'object_' + op
     setattr(cls, op, func)
 
-def add_space_comp_op(cls, op):
-    @raw_binary_op
-    def func(self, v1, v2):
-        return self.space.is_true(getattr(self.space, op)(v1, v2))
-    func.__name__ = 'object_' + op
-    setattr(cls, op, func)
-
 for op in ('copysign', 'isfinite', 'isinf', 'isnan', 'logaddexp', 'logaddexp2',
            'signbit'):
     add_unsupported_op(ObjectType, op)
@@ -1868,8 +1887,6 @@
     add_space_unary_op(ObjectType, op)
 for op in ('add', 'floordiv', 'div', 'mod', 'mul', 'sub', 'lshift', 'rshift'):
     add_space_binary_op(ObjectType, op)
-for op in ('lt', 'gt', 'le', 'ge', 'ne'):
-    add_space_comp_op(ObjectType, op)
 
 ObjectType.fmax = ObjectType.max
 ObjectType.fmin = ObjectType.min

From noreply at buildbot.pypy.org  Mon Apr 13 14:37:04 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 13 Apr 2015 14:37:04 +0200 (CEST)
Subject: [pypy-commit] pypy default: "func.__name__ = ascii_unicode" would
 work on pypy but not on cpython.
Message-ID: <20150413123704.2D5E91C071E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76784:9eec547db5f5
Date: 2015-04-13 14:37 +0200
http://bitbucket.org/pypy/pypy/changeset/9eec547db5f5/

Log:	"func.__name__ = ascii_unicode" would work on pypy but not on
	cpython.

diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -374,14 +374,11 @@
         return space.wrap(self.name)
 
     def fset_func_name(self, space, w_name):
-        try:
+        if space.isinstance_w(w_name, space.w_str):
             self.name = space.str_w(w_name)
-        except OperationError, e:
-            if e.match(space, space.w_TypeError):
-                raise OperationError(space.w_TypeError,
-                                     space.wrap("func_name must be set "
-                                                "to a string object"))
-            raise
+        else:
+            raise OperationError(space.w_TypeError,
+                space.wrap("__name__ must be set to a string object"))
 
     def fdel_func_doc(self, space):
         self.w_doc = space.w_None
diff --git a/pypy/interpreter/test/test_function.py b/pypy/interpreter/test/test_function.py
--- a/pypy/interpreter/test/test_function.py
+++ b/pypy/interpreter/test/test_function.py
@@ -107,6 +107,12 @@
             __name__ = "bar"
             assert f.__module__ == "foo"''' in {}
 
+    def test_set_name(self):
+        def f(): pass
+        f.__name__ = 'g'
+        assert f.func_name == 'g'
+        raises(TypeError, "f.__name__ = u'g'")
+
 
 class AppTestFunction:
     def test_simple_call(self):

From noreply at buildbot.pypy.org  Mon Apr 13 18:07:40 2015
From: noreply at buildbot.pypy.org (antocuni)
Date: Mon, 13 Apr 2015 18:07:40 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: abstract for my EP talk
Message-ID: <20150413160740.3032F1C0ACA@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: extradoc
Changeset: r5526:92dbb3dfe86f
Date: 2015-04-13 18:08 +0200
http://bitbucket.org/pypy/extradoc/changeset/92dbb3dfe86f/

Log:	abstract for my EP talk

diff --git a/talk/ep2015/performance-abstract.txt b/talk/ep2015/performance-abstract.txt
new file mode 100644
--- /dev/null
+++ b/talk/ep2015/performance-abstract.txt
@@ -0,0 +1,20 @@
+==================================
+PyPy performance (not) for dummies
+==================================
+
+
+Abstract
+---------
+
+PyPy is the fastest Python interpreter around, and its JIT can optimize most
+of your Python programs without problems. However, there are techniques to
+improve the performances even further and squeeze the most out of PyPy. In
+this talk we will see:
+
+- the general principles behind the PyPy JIT
+
+- how to profile programs to find the bottlenecks
+
+- how to examine the code generated by the JIT
+
+- how to write JIT-friendly programs

From noreply at buildbot.pypy.org  Mon Apr 13 23:06:47 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 13 Apr 2015 23:06:47 +0200 (CEST)
Subject: [pypy-commit] pypy default: Ignore the recent "movbe" instruction
Message-ID: <20150413210647.C4FF21C00FC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76785:4cadd79e472a
Date: 2015-04-13 23:06 +0200
http://bitbucket.org/pypy/pypy/changeset/4cadd79e472a/

Log:	Ignore the recent "movbe" instruction

diff --git a/rpython/translator/c/gcc/trackgcroot.py b/rpython/translator/c/gcc/trackgcroot.py
--- a/rpython/translator/c/gcc/trackgcroot.py
+++ b/rpython/translator/c/gcc/trackgcroot.py
@@ -526,6 +526,8 @@
         'andn', 'bextr', 'blsi', 'blsmask', 'blsr', 'tzcnt', 'lzcnt',
         # uh, this can occur with a 'call' on the following line...
         'rex64',
+        # movbe, converts from big-endian, so most probably not GC pointers
+        'movbe',
     ])
 
     # a partial list is hopefully good enough for now; it's all to support

From noreply at buildbot.pypy.org  Tue Apr 14 09:18:07 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 14 Apr 2015 09:18:07 +0200 (CEST)
Subject: [pypy-commit] pypy default: issue 2027: pyinteractive.py: add
 --withmod-time, and mention
Message-ID: <20150414071807.DBA311C0369@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76786:ddc09d984319
Date: 2015-04-14 09:18 +0200
http://bitbucket.org/pypy/pypy/changeset/ddc09d984319/

Log:	issue 2027: pyinteractive.py: add --withmod-time, and mention
	--allworkingmodules

diff --git a/pypy/doc/getting-started-dev.rst b/pypy/doc/getting-started-dev.rst
--- a/pypy/doc/getting-started-dev.rst
+++ b/pypy/doc/getting-started-dev.rst
@@ -207,12 +207,17 @@
 large amount of options that can be used to customize pyinteractive.py).
 As an example of using PyPy from the command line, you could type::
 
-    python pyinteractive.py -c "from test import pystone; pystone.main(10)"
+    python pyinteractive.py --withmod-time -c "from test import pystone; pystone.main(10)"
 
 Alternatively, as with regular Python, you can simply give a
 script name on the command line::
 
-    python pyinteractive.py ../../lib-python/2.7/test/pystone.py 10
+    python pyinteractive.py --withmod-time ../../lib-python/2.7/test/pystone.py 10
+
+The ``--withmod-xxx`` option enables the built-in module ``xxx``.  By
+default almost none of them are, because initializing them takes time.
+If you want anyway to enable all built-in modules, you can use
+``--allworkingmodules``.
 
 See our :doc:`configuration sections ` for details about what all the commandline
 options do.

From noreply at buildbot.pypy.org  Tue Apr 14 10:02:26 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 14 Apr 2015 10:02:26 +0200 (CEST)
Subject: [pypy-commit] pypy default: The logic to ask the user for more
 lines was confused by the
Message-ID: <20150414080226.8BDFB1C071E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76787:57faf10f021d
Date: 2015-04-14 10:02 +0200
http://bitbucket.org/pypy/pypy/changeset/57faf10f021d/

Log:	The logic to ask the user for more lines was confused by the
	automatically-inserted indentation, and would ask for one extra line
	since b37ef838fa61.

diff --git a/lib_pypy/pyrepl/simple_interact.py b/lib_pypy/pyrepl/simple_interact.py
--- a/lib_pypy/pyrepl/simple_interact.py
+++ b/lib_pypy/pyrepl/simple_interact.py
@@ -33,6 +33,16 @@
         return False
     return True
 
+def _strip_final_indent(text):
+    # kill spaces and tabs at the end, but only if they follow '\n'.
+    # meant to remove the auto-indentation only (although it would of
+    # course also remove explicitly-added indentation).
+    short = text.rstrip(' \t')
+    n = len(short)
+    if n > 0 and text[n-1] == '\n':
+        return short
+    return text
+
 def run_multiline_interactive_console(mainmodule=None):
     import code
     if mainmodule is None:
@@ -41,7 +51,7 @@
 
     def more_lines(unicodetext):
         # ooh, look at the hack:
-        src = "#coding:utf-8\n"+unicodetext.encode('utf-8')
+        src = "#coding:utf-8\n"+_strip_final_indent(unicodetext).encode('utf-8')
         try:
             code = console.compile(src, '', 'single')
         except (OverflowError, SyntaxError, ValueError):
@@ -58,7 +68,7 @@
                                             returns_unicode=True)
             except EOFError:
                 break
-            more = console.push(statement)
+            more = console.push(_strip_final_indent(statement))
             assert not more
         except KeyboardInterrupt:
             console.write("\nKeyboardInterrupt\n")

From noreply at buildbot.pypy.org  Tue Apr 14 10:24:55 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 14 Apr 2015 10:24:55 +0200 (CEST)
Subject: [pypy-commit] pypy default: remove a print which causes occasional
 confusion in the app multithreaded test
Message-ID: <20150414082455.8DC1F1C0369@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76788:67a09bbd332a
Date: 2015-04-14 10:24 +0200
http://bitbucket.org/pypy/pypy/changeset/67a09bbd332a/

Log:	remove a print which causes occasional confusion in the app
	multithreaded test

diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -642,7 +642,7 @@
 
         thread_id = thread.get_ident()
         def other_thread():
-            print "thread started"
+            #print "thread started"
             lock2.release()
             lock1.acquire()
         lock1 = thread.allocate_lock()

From noreply at buildbot.pypy.org  Tue Apr 14 13:01:38 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 14 Apr 2015 13:01:38 +0200 (CEST)
Subject: [pypy-commit] pypy default: issue 2029: try a bit harder to hide
 the "default_factory" attribute
Message-ID: <20150414110138.275751C071E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76789:494280f37b17
Date: 2015-04-14 13:01 +0200
http://bitbucket.org/pypy/pypy/changeset/494280f37b17/

Log:	issue 2029: try a bit harder to hide the "default_factory" attribute

diff --git a/pypy/module/_collections/app_defaultdict.py b/pypy/module/_collections/app_defaultdict.py
--- a/pypy/module/_collections/app_defaultdict.py
+++ b/pypy/module/_collections/app_defaultdict.py
@@ -11,6 +11,7 @@
 
 
 class defaultdict(dict):
+    __slots__ = ['__default_factory']
 
     def __init__(self, *args, **kwds):
         if len(args) > 0:
@@ -20,7 +21,7 @@
                 raise TypeError("first argument must be callable")
         else:
             default_factory = None
-        self.default_factory = default_factory
+        defaultdict.__default_factory.__set__(self, default_factory)
         super(defaultdict, self).__init__(*args, **kwds)
 
     def __missing__(self, key):
@@ -33,15 +34,15 @@
             return "defaultdict(...)"
         try:
             recurse.add(id(self))
-            return "defaultdict(%s, %s)" % (repr(self.default_factory), super(defaultdict, self).__repr__())
+            return "defaultdict(%s, %s)" % (self.__default_factory,
+                                            super(defaultdict, self).__repr__())
         finally:
             recurse.remove(id(self))
 
     def copy(self):
-        return type(self)(self.default_factory, self)
+        return type(self)(self.__default_factory, self)
 
-    def __copy__(self):
-        return self.copy()
+    __copy__ = copy
 
     def __reduce__(self):
         """
@@ -55,4 +56,5 @@
 
            This API is used by pickle.py and copy.py.
         """
-        return (type(self), (self.default_factory,), None, None, self.iteritems())
+        return (type(self), (self.__default_factory,), None, None,
+                defaultdict.iteritems(self))
diff --git a/pypy/module/_collections/interp_defaultdict.py b/pypy/module/_collections/interp_defaultdict.py
--- a/pypy/module/_collections/interp_defaultdict.py
+++ b/pypy/module/_collections/interp_defaultdict.py
@@ -4,7 +4,8 @@
     # An interp-level version of this method.  This is mostly only
     # useful because it can be executed atomically in the presence of
     # threads.
-    w_default_factory = space.getattr(w_self, space.wrap('default_factory'))
+    w_default_factory = space.getattr(w_self,
+            space.wrap('_defaultdict__default_factory'))
     if space.is_w(w_default_factory, space.w_None):
         raise OperationError(space.w_KeyError, space.newtuple([w_key]))
     w_value = space.call_function(w_default_factory)
diff --git a/pypy/module/_collections/test/test_defaultdict.py b/pypy/module/_collections/test/test_defaultdict.py
--- a/pypy/module/_collections/test/test_defaultdict.py
+++ b/pypy/module/_collections/test/test_defaultdict.py
@@ -54,3 +54,17 @@
         assert len(d2) == 1
         assert d2[2] == 3
         assert d2[3] == 42
+
+    def test_no_dict(self):
+        import _collections
+        assert not hasattr(_collections.defaultdict(), '__dict__')
+
+    def test_no_setattr(self):
+        import _collections
+        class D(_collections.defaultdict):
+            def __setattr__(self, attr, name):
+                raise AssertionError
+        d = D(int)
+        assert d['5'] == 0
+        d['6'] += 3
+        assert d['6'] == 3

From noreply at buildbot.pypy.org  Tue Apr 14 13:31:22 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 14 Apr 2015 13:31:22 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: rework the abstract
Message-ID: <20150414113122.C0BA61C06CD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: extradoc
Changeset: r5527:1e9cb1d73547
Date: 2015-04-14 13:31 +0200
http://bitbucket.org/pypy/extradoc/changeset/1e9cb1d73547/

Log:	rework the abstract

diff --git a/talk/ep2015/performance-abstract.txt b/talk/ep2015/performance-abstract.txt
--- a/talk/ep2015/performance-abstract.txt
+++ b/talk/ep2015/performance-abstract.txt
@@ -1,20 +1,21 @@
-==================================
-PyPy performance (not) for dummies
-==================================
+====================================
+Python performance (not) for dummies
+====================================
 
+In this talk we would like to have a short introduction on how Python
+programs are compiled and executed, with a special attention towards
+just in time compilation done by PyPy. PyPy is the most advanced Python
+interpreter around and while it should generally just speed up your programs
+there is a wide range of performance that you can get out of PyPy, ranging from
+slightly faster than CPython to C speeds, depending on how you write your
+programs.
 
-Abstract
----------
+We will split the talk in two parts. In the first part we will explain
+how things work and what can and what cannot be optimized as well as describe
+the basic heuristics of JIT compiler and optimizer. In the next part we will
+do a survey of existing tools for looking at performance of Python programs
+with specific focus on PyPy.
 
-PyPy is the fastest Python interpreter around, and its JIT can optimize most
-of your Python programs without problems. However, there are techniques to
-improve the performances even further and squeeze the most out of PyPy. In
-this talk we will see:
-
-- the general principles behind the PyPy JIT
-
-- how to profile programs to find the bottlenecks
-
-- how to examine the code generated by the JIT
-
-- how to write JIT-friendly programs
+As a result of this talk, an audience member should be better equipped with
+tools how to write new software and improve existing software with performance
+in mind.

From noreply at buildbot.pypy.org  Tue Apr 14 14:29:59 2015
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue, 14 Apr 2015 14:29:59 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: tweaks
Message-ID: <20150414122959.777401C0661@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: extradoc
Changeset: r5528:5bb398128d9a
Date: 2015-04-14 12:30 +0000
http://bitbucket.org/pypy/extradoc/changeset/5bb398128d9a/

Log:	tweaks

diff --git a/talk/ep2015/performance-abstract.txt b/talk/ep2015/performance-abstract.txt
--- a/talk/ep2015/performance-abstract.txt
+++ b/talk/ep2015/performance-abstract.txt
@@ -1,6 +1,10 @@
-====================================
-Python performance (not) for dummies
-====================================
+==============================================
+Python and PyPy performance (not) for dummies
+==============================================
+
+[[The talk will be given by Antonio Cuni and Maciej Fijalkowski,
+both long time PyPy core developers and expert in the area of
+Python performance.]]
 
 In this talk we would like to have a short introduction on how Python
 programs are compiled and executed, with a special attention towards

From noreply at buildbot.pypy.org  Tue Apr 14 15:35:21 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 14 Apr 2015 15:35:21 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Use uintptr_t instead of unsigned long
Message-ID: <20150414133521.417671C1201@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1696:ee25743d7af9
Date: 2015-04-11 20:33 +0200
http://bitbucket.org/cffi/cffi/changeset/ee25743d7af9/

Log:	Use uintptr_t instead of unsigned long

diff --git a/new/parse_c_type.c b/new/parse_c_type.c
--- a/new/parse_c_type.c
+++ b/new/parse_c_type.c
@@ -45,7 +45,7 @@
     enum token_e kind;
     size_t size;
     _cffi_opcode_t *output;
-    unsigned long output_index;
+    size_t output_index;
 } token_t;
 
 static int is_space(char x)
diff --git a/new/parse_c_type.h b/new/parse_c_type.h
--- a/new/parse_c_type.h
+++ b/new/parse_c_type.h
@@ -3,9 +3,9 @@
 
 typedef void *_cffi_opcode_t;
 
-#define _CFFI_OP(opcode, arg)   (_cffi_opcode_t)(opcode | (((unsigned long)(arg)) << 8))
-#define _CFFI_GETOP(cffi_opcode)    ((unsigned char)(unsigned long)cffi_opcode)
-#define _CFFI_GETARG(cffi_opcode)   (((unsigned long)cffi_opcode) >> 8)
+#define _CFFI_OP(opcode, arg)   (_cffi_opcode_t)(opcode | (((uintptr_t)(arg)) << 8))
+#define _CFFI_GETOP(cffi_opcode)    ((unsigned char)(uintptr_t)cffi_opcode)
+#define _CFFI_GETARG(cffi_opcode)   (((uintptr_t)cffi_opcode) >> 8)
 
 #define _CFFI_OP_PRIMITIVE       1
 #define _CFFI_OP_POINTER         3

From noreply at buildbot.pypy.org  Tue Apr 14 15:35:22 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 14 Apr 2015 15:35:22 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: fix test
Message-ID: <20150414133522.6F3531C1201@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1697:acc5959cd6a8
Date: 2015-04-11 20:39 +0200
http://bitbucket.org/cffi/cffi/changeset/acc5959cd6a8/

Log:	fix test

diff --git a/new/parse_c_type.h b/new/parse_c_type.h
--- a/new/parse_c_type.h
+++ b/new/parse_c_type.h
@@ -1,4 +1,4 @@
-#include 
+#include 
 
 
 typedef void *_cffi_opcode_t;
diff --git a/new/realize_c_type.c b/new/realize_c_type.c
--- a/new/realize_c_type.c
+++ b/new/realize_c_type.c
@@ -34,6 +34,11 @@
 {
     PyObject *x, *y;
     _cffi_opcode_t op = opcodes[index];
+    if ((((uintptr_t)op) & 1) == 0) {
+        x = (PyObject *)op;
+        Py_INCREF(x);
+        return x;
+    }
 
     switch (_CFFI_GETOP(op)) {
 
@@ -42,7 +47,7 @@
         if (x == NULL)
             x = build_primitive_type(_CFFI_GETARG(op));
         Py_XINCREF(x);
-        return x;
+        break;
 
     case _CFFI_OP_POINTER:
         y = realize_c_type(ctx, opcodes, _CFFI_GETARG(op));
@@ -50,16 +55,18 @@
             return NULL;
         x = Py_BuildValue("sO", "pointer", y);
         Py_DECREF(y);
-        return x;
+        break;
 
     default:
         PyErr_Format(PyExc_NotImplementedError, "op=%d", (int)_CFFI_GETOP(op));
         return NULL;
     }
-}
 
-
-struct _cffi_type_context_s global_ctx = {
+    if (opcodes == ctx->types) {
+        Py_INCREF(x);
+        opcodes[index] = x;
+    }
+    return x;
 };
 
 
@@ -70,6 +77,7 @@
         return NULL;
 
     _cffi_opcode_t opcodes[100];
+    struct _cffi_type_context_s global_ctx = { NULL };
     struct _cffi_parse_info_s parse_info = {
         .ctx = &global_ctx,
         .output = opcodes,
diff --git a/new/test_realize_c_type.py b/new/test_realize_c_type.py
--- a/new/test_realize_c_type.py
+++ b/new/test_realize_c_type.py
@@ -1,7 +1,9 @@
 import os
 
 def setup_module():
-    os.system("python setup.py build_ext -i")
+    err = os.system("python setup.py build_ext -i")
+    if err != 0:
+        raise Exception(err)
     global realize_c_type
     import realize_c_type
 

From noreply at buildbot.pypy.org  Tue Apr 14 15:35:23 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 14 Apr 2015 15:35:23 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: More tweaks
Message-ID: <20150414133523.7F7CE1C1201@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1698:52c9ac54d677
Date: 2015-04-12 10:23 +0200
http://bitbucket.org/cffi/cffi/changeset/52c9ac54d677/

Log:	More tweaks

diff --git a/new/parse_c_type.h b/new/parse_c_type.h
--- a/new/parse_c_type.h
+++ b/new/parse_c_type.h
@@ -17,6 +17,8 @@
 #define _CFFI_OP_FUNCTION       15
 #define _CFFI_OP_FUNCTION_END   17
 #define _CFFI_OP_NOOP           19
+#define _CFFI_OP_BITFIELD       21
+#define _CFFI_OP_CPYTHON_BLTN   23
 
 #define _CFFI_PRIM_VOID          0
 #define _CFFI_PRIM_BOOL          1
@@ -40,16 +42,14 @@
 struct _cffi_global_s {
     const char *name;
     void *address;
-    int type_index;
+    _cffi_opcode_t type_op;
 };
 
 struct _cffi_constant_s {
     const char *name;
     unsigned long long value;
-    int type_index;          // -> _cffi_types or _CFFI_PLAIN_*_INT
+    _cffi_opcode_t type_op;
 };
-#define _CFFI_PLAIN_POSITIVE_INT     (-1)
-#define _CFFI_PLAIN_NONPOSITIVE_INT  (-2)
 
 struct _cffi_struct_union_s {
     const char *name;
@@ -66,18 +66,17 @@
     const char *name;
     size_t field_offset;
     size_t field_size;
-    int field_bit_size;
-    int field_type;          // -> _cffi_types
+    _cffi_opcode_t field_type_op;
 };
 
 struct _cffi_enum_s {
     const char *name;
-    int integer_type;        // -> _cffi_types
+    _cffi_opcode_t integer_type_op;
 };
 
 struct _cffi_typename_s {
     const char *name;
-    int type_index;          // -> _cffi_types
+    _cffi_opcode_t type_op;   /* 0 if opaque or prebuilt */
 };
 
 struct _cffi_type_context_s {

From noreply at buildbot.pypy.org  Tue Apr 14 15:35:24 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 14 Apr 2015 15:35:24 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: in-progress
Message-ID: <20150414133524.9CB471C1201@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1699:943c257b5873
Date: 2015-04-14 15:35 +0200
http://bitbucket.org/cffi/cffi/changeset/943c257b5873/

Log:	in-progress

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -3338,7 +3338,7 @@
 
 /************************************************************/
 
-static PyObject *b_new_primitive_type(PyObject *self, PyObject *args)
+static CTypeDescrObject *new_primitive_type(const char *name)
 {
 #define ENUM_PRIMITIVE_TYPES                                    \
        EPTYPE(c, char, CT_PRIMITIVE_CHAR)                       \
@@ -3403,7 +3403,6 @@
 #undef EPTYPE
 
     CTypeDescrObject *td;
-    const char *name;
     static const struct descr_s { const char *name; int size, align, flags; }
     types[] = {
 #define EPTYPE(code, typename, flags)                   \
@@ -3422,9 +3421,6 @@
     int name_size;
     ffi_type *ffitype;
 
-    if (!PyArg_ParseTuple(args, "s:new_primitive_type", &name))
-        return NULL;
-
     for (ptypes=types; ; ptypes++) {
         if (ptypes->name == NULL) {
 #ifndef HAVE_WCHAR_H
@@ -3487,7 +3483,7 @@
             td->ct_flags |= CT_PRIMITIVE_FITS_LONG;
     }
     td->ct_name_position = strlen(td->ct_name);
-    return (PyObject *)td;
+    return td;
 
  bad_ffi_type:
     PyErr_Format(PyExc_NotImplementedError,
@@ -3497,15 +3493,19 @@
     return NULL;
 }
 
-static PyObject *b_new_pointer_type(PyObject *self, PyObject *args)
-{
-    CTypeDescrObject *td, *ctitem;
+static PyObject *b_new_primitive_type(PyObject *self, PyObject *args)
+{
+    char *name;
+    if (!PyArg_ParseTuple(args, "s:new_primitive_type", &name))
+        return NULL;
+    return (PyObject *)new_primitive_type(name);
+}
+
+static CTypeDescrObject *new_pointer_type(CTypeDescrObject *ctitem)
+{
+    CTypeDescrObject *td;
     const char *extra;
 
-    if (!PyArg_ParseTuple(args, "O!:new_pointer_type",
-                          &CTypeDescr_Type, &ctitem))
-        return NULL;
-
     if (ctitem->ct_flags & CT_ARRAY)
         extra = "(*)";   /* obscure case: see test_array_add */
     else
@@ -3525,7 +3525,16 @@
         ((ctitem->ct_flags & CT_PRIMITIVE_CHAR) &&
          ctitem->ct_size == sizeof(char)))
         td->ct_flags |= CT_CAST_ANYTHING;   /* 'void *' or 'char *' only */
-    return (PyObject *)td;
+    return td;
+}
+
+static PyObject *b_new_pointer_type(PyObject *self, PyObject *args)
+{
+    CTypeDescrObject *ctitem;
+    if (!PyArg_ParseTuple(args, "O!:new_pointer_type",
+                          &CTypeDescr_Type, &ctitem))
+        return NULL;
+    return (PyObject *)new_pointer_type(ctitem);
 }
 
 static PyObject *b_new_array_type(PyObject *self, PyObject *args)
@@ -3594,7 +3603,7 @@
     return (PyObject *)td;
 }
 
-static PyObject *b_new_void_type(PyObject *self, PyObject *args)
+static CTypeDescrObject *new_void_type(void)
 {
     int name_size = strlen("void") + 1;
     CTypeDescrObject *td = ctypedescr_new(name_size);
@@ -3605,7 +3614,12 @@
     td->ct_size = -1;
     td->ct_flags = CT_VOID | CT_IS_OPAQUE;
     td->ct_name_position = strlen("void");
-    return (PyObject *)td;
+    return td;
+}
+
+static PyObject *b_new_void_type(PyObject *self, PyObject *args)
+{
+    return (PyObject *)new_void_type();
 }
 
 static PyObject *_b_struct_or_union_type(const char *name, int flag)
@@ -5683,8 +5697,10 @@
 }
 #endif
 
+#include "../new/cffi1_module.c"
+
 static void *cffi_exports[] = {
-    0,
+    (void *)26,
     _cffi_to_c_i8,
     _cffi_to_c_u8,
     _cffi_to_c_i16,
@@ -5714,6 +5730,7 @@
     _cffi_to_c__Bool,
     _prepare_pointer_call_argument,
     convert_array_from_object,
+    _cffi_init_module,
 };
 
 /************************************************************/
diff --git a/new/_cffi_include.h b/new/_cffi_include.h
new file mode 100644
--- /dev/null
+++ b/new/_cffi_include.h
@@ -0,0 +1,191 @@
+#include 
+#include 
+#include "parse_c_type.h"
+
+/* this block of #ifs should be kept exactly identical between
+   c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py */
+#if defined(_MSC_VER)
+# include    /* for alloca() */
+# if _MSC_VER < 1600   /* MSVC < 2010 */
+   typedef __int8 int8_t;
+   typedef __int16 int16_t;
+   typedef __int32 int32_t;
+   typedef __int64 int64_t;
+   typedef unsigned __int8 uint8_t;
+   typedef unsigned __int16 uint16_t;
+   typedef unsigned __int32 uint32_t;
+   typedef unsigned __int64 uint64_t;
+   typedef __int8 int_least8_t;
+   typedef __int16 int_least16_t;
+   typedef __int32 int_least32_t;
+   typedef __int64 int_least64_t;
+   typedef unsigned __int8 uint_least8_t;
+   typedef unsigned __int16 uint_least16_t;
+   typedef unsigned __int32 uint_least32_t;
+   typedef unsigned __int64 uint_least64_t;
+   typedef __int8 int_fast8_t;
+   typedef __int16 int_fast16_t;
+   typedef __int32 int_fast32_t;
+   typedef __int64 int_fast64_t;
+   typedef unsigned __int8 uint_fast8_t;
+   typedef unsigned __int16 uint_fast16_t;
+   typedef unsigned __int32 uint_fast32_t;
+   typedef unsigned __int64 uint_fast64_t;
+   typedef __int64 intmax_t;
+   typedef unsigned __int64 uintmax_t;
+# else
+#  include 
+# endif
+# if _MSC_VER < 1800   /* MSVC < 2013 */
+   typedef unsigned char _Bool;
+# endif
+#else
+# include 
+# if (defined (__SVR4) && defined (__sun)) || defined(_AIX)
+#  include 
+# endif
+#endif
+
+#if PY_MAJOR_VERSION < 3
+# undef PyCapsule_CheckExact
+# undef PyCapsule_GetPointer
+# define PyCapsule_CheckExact(capsule) (PyCObject_Check(capsule))
+# define PyCapsule_GetPointer(capsule, name) \
+    (PyCObject_AsVoidPtr(capsule))
+#endif
+
+#if PY_MAJOR_VERSION >= 3
+# define PyInt_FromLong PyLong_FromLong
+#endif
+
+#define _cffi_from_c_double PyFloat_FromDouble
+#define _cffi_from_c_float PyFloat_FromDouble
+#define _cffi_from_c_long PyInt_FromLong
+#define _cffi_from_c_ulong PyLong_FromUnsignedLong
+#define _cffi_from_c_longlong PyLong_FromLongLong
+#define _cffi_from_c_ulonglong PyLong_FromUnsignedLongLong
+
+#define _cffi_to_c_double PyFloat_AsDouble
+#define _cffi_to_c_float PyFloat_AsDouble
+
+#define _cffi_from_c_int_const(x)                                        \
+    (((x) > 0) ?                                                         \
+        ((unsigned long long)(x) <= (unsigned long long)LONG_MAX) ?      \
+            PyInt_FromLong((long)(x)) :                                  \
+            PyLong_FromUnsignedLongLong((unsigned long long)(x)) :       \
+        ((long long)(x) >= (long long)LONG_MIN) ?                        \
+            PyInt_FromLong((long)(x)) :                                  \
+            PyLong_FromLongLong((long long)(x)))
+
+#define _cffi_from_c_int(x, type)                                        \
+    (((type)-1) > 0 ? /* unsigned */                                     \
+        (sizeof(type) < sizeof(long) ?                                   \
+            PyInt_FromLong((long)x) :                                    \
+         sizeof(type) == sizeof(long) ?                                  \
+            PyLong_FromUnsignedLong((unsigned long)x) :                  \
+            PyLong_FromUnsignedLongLong((unsigned long long)x)) :        \
+        (sizeof(type) <= sizeof(long) ?                                  \
+            PyInt_FromLong((long)x) :                                    \
+            PyLong_FromLongLong((long long)x)))
+
+#define _cffi_to_c_int(o, type)                                          \
+    (sizeof(type) == 1 ? (((type)-1) > 0 ? (type)_cffi_to_c_u8(o)        \
+                                         : (type)_cffi_to_c_i8(o)) :     \
+     sizeof(type) == 2 ? (((type)-1) > 0 ? (type)_cffi_to_c_u16(o)       \
+                                         : (type)_cffi_to_c_i16(o)) :    \
+     sizeof(type) == 4 ? (((type)-1) > 0 ? (type)_cffi_to_c_u32(o)       \
+                                         : (type)_cffi_to_c_i32(o)) :    \
+     sizeof(type) == 8 ? (((type)-1) > 0 ? (type)_cffi_to_c_u64(o)       \
+                                         : (type)_cffi_to_c_i64(o)) :    \
+     (Py_FatalError("unsupported size for type " #type), (type)0))
+
+#define _cffi_to_c_i8                                                    \
+                 ((int(*)(PyObject *))_cffi_exports[1])
+#define _cffi_to_c_u8                                                    \
+                 ((int(*)(PyObject *))_cffi_exports[2])
+#define _cffi_to_c_i16                                                   \
+                 ((int(*)(PyObject *))_cffi_exports[3])
+#define _cffi_to_c_u16                                                   \
+                 ((int(*)(PyObject *))_cffi_exports[4])
+#define _cffi_to_c_i32                                                   \
+                 ((int(*)(PyObject *))_cffi_exports[5])
+#define _cffi_to_c_u32                                                   \
+                 ((unsigned int(*)(PyObject *))_cffi_exports[6])
+#define _cffi_to_c_i64                                                   \
+                 ((long long(*)(PyObject *))_cffi_exports[7])
+#define _cffi_to_c_u64                                                   \
+                 ((unsigned long long(*)(PyObject *))_cffi_exports[8])
+#define _cffi_to_c_char                                                  \
+                 ((int(*)(PyObject *))_cffi_exports[9])
+#define _cffi_from_c_pointer                                             \
+    ((PyObject *(*)(char *, CTypeDescrObject *))_cffi_exports[10])
+#define _cffi_to_c_pointer                                               \
+    ((char *(*)(PyObject *, CTypeDescrObject *))_cffi_exports[11])
+#define _cffi_get_struct_layout                                          \
+    ((PyObject *(*)(Py_ssize_t[]))_cffi_exports[12])
+#define _cffi_restore_errno                                              \
+    ((void(*)(void))_cffi_exports[13])
+#define _cffi_save_errno                                                 \
+    ((void(*)(void))_cffi_exports[14])
+#define _cffi_from_c_char                                                \
+    ((PyObject *(*)(char))_cffi_exports[15])
+#define _cffi_from_c_deref                                               \
+    ((PyObject *(*)(char *, CTypeDescrObject *))_cffi_exports[16])
+#define _cffi_to_c                                                       \
+    ((int(*)(char *, CTypeDescrObject *, PyObject *))_cffi_exports[17])
+#define _cffi_from_c_struct                                              \
+    ((PyObject *(*)(char *, CTypeDescrObject *))_cffi_exports[18])
+#define _cffi_to_c_wchar_t                                               \
+    ((wchar_t(*)(PyObject *))_cffi_exports[19])
+#define _cffi_from_c_wchar_t                                             \
+    ((PyObject *(*)(wchar_t))_cffi_exports[20])
+#define _cffi_to_c_long_double                                           \
+    ((long double(*)(PyObject *))_cffi_exports[21])
+#define _cffi_to_c__Bool                                                 \
+    ((_Bool(*)(PyObject *))_cffi_exports[22])
+#define _cffi_prepare_pointer_call_argument                              \
+    ((Py_ssize_t(*)(CTypeDescrObject *, PyObject *, char **))_cffi_exports[23])
+#define _cffi_convert_array_from_object                                  \
+    ((int(*)(char *, CTypeDescrObject *, PyObject *))_cffi_exports[24])
+#define _cffi_init_module                                                \
+    ((int(*)(char *, const struct _cffi_type_context_s *))_cffi_exports[25])
+#define _CFFI_NUM_EXPORTS 26
+
+typedef struct _ctypedescr CTypeDescrObject;
+
+static void *_cffi_exports[_CFFI_NUM_EXPORTS];
+
+
+static int _cffi_init(void)
+{
+    PyObject *module, *c_api_object = NULL;
+    void *src;
+
+    module = PyImport_ImportModule("_cffi_backend");
+    if (module == NULL)
+        goto failure;
+
+    c_api_object = PyObject_GetAttrString(module, "_C_API");
+    if (c_api_object == NULL)
+        goto failure;
+    if (!PyCapsule_CheckExact(c_api_object)) {
+        PyErr_SetNone(PyExc_ImportError);
+        goto failure;
+    }
+    src = PyCapsule_GetPointer(c_api_object, "cffi");
+    if ((uintptr_t)(((void **)src)[0]) < _CFFI_NUM_EXPORTS) {
+        PyErr_SetString(PyExc_ImportError,
+                        "the _cffi_backend module is an outdated version");
+        goto failure;
+    }
+    memcpy(_cffi_exports, src, _CFFI_NUM_EXPORTS * sizeof(void *));
+
+    Py_DECREF(module);
+    Py_DECREF(c_api_object);
+    return 0;
+
+  failure:
+    Py_XDECREF(module);
+    Py_XDECREF(c_api_object);
+    return -1;
+}
diff --git a/new/cffi1_module.c b/new/cffi1_module.c
new file mode 100644
--- /dev/null
+++ b/new/cffi1_module.c
@@ -0,0 +1,52 @@
+#include "parse_c_type.c"
+#include "realize_c_type.c"
+
+typedef struct FFIObject_s FFIObject;
+typedef struct LibObject_s LibObject;
+
+static PyTypeObject FFI_Type;   /* forward */
+static PyTypeObject Lib_Type;   /* forward */
+
+static PyObject *FFIError;
+
+#include "ffi_obj.c"
+//#include "lib_obj.c"
+
+
+static int init_ffi_lib(void)
+{
+    static int ffi_lib_ok = 0;
+    if (ffi_lib_ok)
+        return 0;
+
+    if (!PyType_Ready(&FFI_Type) < 0)
+        return -1;
+    if (!PyType_Ready(&Lib_Type) < 0)
+        return -1;
+
+    FFIError = PyErr_NewException("ffi.error", NULL, NULL);
+    if (FFIError == NULL)
+        return -1;
+    if (PyDict_SetItemString(FFI_Type.tp_dict, "error", FFIError) < 0)
+        return -1;
+
+    ffi_lib_ok = 1;
+    return 0;
+}
+
+static int _cffi_init_module(char *module_name,
+                             const struct _cffi_type_context_s *ctx)
+{
+    if (init_ffi_lib() < 0)
+        return -1;
+
+    PyObject *m = Py_InitModule(module_name, NULL);
+    if (m == NULL)
+        return -1;
+
+    FFIObject *ffi = ffi_internal_new(ctx);
+    if (ffi == NULL || PyModule_AddObject(m, "ffi", (PyObject *)ffi) < 0)
+        return -1;
+
+    return 0;
+}
diff --git a/new/ffi_obj.c b/new/ffi_obj.c
new file mode 100644
--- /dev/null
+++ b/new/ffi_obj.c
@@ -0,0 +1,603 @@
+
+/* An FFI object has methods like ffi.new().  It is also a container
+   for the type declarations (typedefs and structs) that you can use,
+   say in ffi.new().
+
+   CTypeDescrObjects are internally stored in the dict 'types_dict'.
+   The types_dict is lazily filled with CTypeDescrObjects made from
+   reading a _cffi_type_context_s structure.
+
+   In "modern" mode, the FFI instance is made by the C extension
+   module originally created by recompile().  The _cffi_type_context_s
+   structure comes from global data in the C extension module.
+
+   In "compatibility" mode, an FFI instance is created explicitly by
+   the user, and its _cffi_type_context_s is initially empty.  You
+   need to call ffi.cdef() to add more information to it.
+*/
+
+#define FFI_COMPLEXITY_OUTPUT   1200     /* xxx should grow as needed */
+
+struct FFIObject_s {
+    PyObject_HEAD
+    PyObject *types_dict;
+    PyObject *gc_wrefs;
+    struct _cffi_parse_info_s info;
+    _cffi_opcode_t internal_output[FFI_COMPLEXITY_OUTPUT];
+};
+
+static FFIObject *ffi_internal_new(const struct _cffi_type_context_s *ctx)
+{
+    PyObject *dict = PyDict_New();
+    if (dict == NULL)
+        return NULL;
+
+    FFIObject *ffi = PyObject_GC_New(FFIObject, &FFI_Type);
+    if (ffi == NULL) {
+        Py_DECREF(dict);
+        return NULL;
+    }
+    ffi->types_dict = dict;
+    ffi->gc_wrefs = NULL;
+    ffi->info.ctx = ctx;
+    ffi->info.output = ffi->internal_output;
+    ffi->info.output_size = FFI_COMPLEXITY_OUTPUT;
+
+    PyObject_GC_Track(ffi);
+    return ffi;
+}
+
+static void ffi_dealloc(FFIObject *ffi)
+{
+    PyObject_GC_UnTrack(ffi);
+    Py_DECREF(ffi->types_dict);
+    Py_XDECREF(ffi->gc_wrefs);
+    PyObject_GC_Del(ffi);
+}
+
+static int ffi_traverse(FFIObject *ffi, visitproc visit, void *arg)
+{
+    Py_VISIT(ffi->types_dict);
+    Py_VISIT(ffi->gc_wrefs);
+    return 0;
+}
+
+static PyObject *ffiobj_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+{
+    /* user-facing initialization code, for explicit FFI() calls */
+    /* XXX LATER */
+    PyErr_SetString(PyExc_NotImplementedError, "FFI()");
+    return NULL;
+}
+
+#define ACCEPT_STRING   1
+#define ACCEPT_CTYPE    2
+#define ACCEPT_CDATA    4
+#define ACCEPT_ALL      (ACCEPT_STRING | ACCEPT_CTYPE | ACCEPT_CDATA)
+
+static CTypeDescrObject *_ffi_type(FFIObject *ffi, PyObject *arg,
+                                   int accept)
+{
+    /* Returns the CTypeDescrObject from the user-supplied 'arg'.
+       Does not return a new reference!
+    */
+    if ((accept & ACCEPT_STRING) && PyText_Check(arg)) {
+        PyObject *x = PyDict_GetItem(ffi->types_dict, arg);
+        if (x != NULL && CTypeDescr_Check(x))
+            return (CTypeDescrObject *)x;
+
+        char *input_text = PyText_AS_UTF8(arg);
+        int index = parse_c_type(&ffi->info, input_text);
+        if (index < 0) {
+            size_t num_spaces = ffi->info.error_location;
+            char spaces[num_spaces + 1];
+            memset(spaces, ' ', num_spaces);
+            spaces[num_spaces] = '\0';
+            PyErr_Format(FFIError, "%s\n%s\n%s^", ffi->info.error_message,
+                         input_text, spaces);
+            return NULL;
+        }
+        CTypeDescrObject *ct = realize_c_type(ffi->info.ctx,
+                                              ffi->info.output, index);
+        if (ct == NULL)
+            return NULL;
+        x = PyDict_GetItemString(ffi->types_dict, ct->ct_name);
+        if (x != NULL) {
+            Py_INCREF(x);
+            Py_DECREF(ct);
+            ct = (CTypeDescrObject *)x;
+        }
+        PyDict_SetItemString(ffi->types_dict, ct->ct_name, (PyObject *)ct);
+        PyDict_SetItem(ffi->types_dict, arg, (PyObject *)ct);
+        return ct;
+    }
+    else if ((accept & ACCEPT_CTYPE) && CTypeDescr_Check(arg)) {
+        return (CTypeDescrObject *)arg;
+    }
+    else if ((accept & ACCEPT_CDATA) && CData_Check(arg)) {
+        return ((CDataObject *)arg)->c_type;
+    }
+    else {
+        const char *m1 = (accept & ACCEPT_STRING) ? "string" : "";
+        const char *m2 = (accept & ACCEPT_CTYPE) ? "ctype object" : "";
+        const char *m3 = (accept & ACCEPT_CDATA) ? "cdata object" : "";
+        const char *s12 = (*m1 && (*m2 || *m3)) ? " or " : "";
+        const char *s23 = (*m2 && *m3) ? " or " : "";
+        PyErr_Format(PyExc_TypeError, "expected a %s%s%s%s%s, got '%.200s'",
+                     m1, s12, m2, s23, m3,
+                     Py_TYPE(arg)->tp_name);
+        return NULL;
+    }
+}
+
+static PyObject *ffi_sizeof(FFIObject *self, PyObject *arg)
+{
+    CTypeDescrObject *ct = _ffi_type(self, arg, ACCEPT_ALL);
+    if (ct == NULL)
+        return NULL;
+
+    if (ct->ct_size < 0) {
+        PyErr_Format(FFIError, "don't know the size of ctype '%s'",
+                     ct->ct_name);
+        return NULL;
+    }
+    return PyInt_FromSsize_t(ct->ct_size);
+}
+
+#if 0
+static PyObject *ffi_typeof(ZefFFIObject *self, PyObject *arg)
+{
+    PyObject *x = (PyObject *)_ffi_type(self, arg, ACCEPT_STRING|ACCEPT_CDATA);
+    if (x != NULL) {
+        Py_INCREF(x);
+    }
+    else if (PyCFunction_Check(arg)) {
+        PyErr_SetString(PyExc_TypeError, "typeof(lib.func) not supported: the "
+                        "exact type of functions is unknown (declare that "
+                        "function as a function pointer instead)");
+    }
+    return x;
+}
+
+static PyObject *ffi_new(ZefFFIObject *self, PyObject *args)
+{
+    CTypeDescrObject *ct, *ctitem;
+    CDataObject *cd;
+    PyObject *arg, *init = Py_None;
+    Py_ssize_t dataoffset, datasize, explicitlength;
+    if (!PyArg_ParseTuple(args, "O|O:new", &arg, &init))
+        return NULL;
+
+    ct = _ffi_type(self, arg, ACCEPT_STRING|ACCEPT_CTYPE);
+    if (ct == NULL)
+        return NULL;
+
+    explicitlength = -1;
+    if (ct->ct_flags & (CT_POINTER | CT_STRUCT | CT_UNION)) {
+        dataoffset = offsetof(CDataObject_own_nolength, alignment);
+        ctitem = (ct->ct_flags & CT_POINTER) ? ct->ct_itemdescr : ct;
+        datasize = ctitem->ct_size;
+        if (datasize < 0) {
+            PyErr_Format(PyExc_TypeError,
+                         "cannot instantiate ctype '%s' of unknown size",
+                         ctitem->ct_name);
+            return NULL;
+        }
+        if (ctitem->ct_flags & CT_PRIMITIVE_CHAR)
+            datasize *= 2;   /* forcefully add another character: a null */
+
+        if ((ctitem->ct_flags & CT_WITH_VAR_ARRAY) && init != Py_None) {
+            Py_ssize_t optvarsize = datasize;
+            if (convert_struct_from_object(NULL,ctitem, init, &optvarsize) < 0)
+                return NULL;
+            datasize = optvarsize;
+        }
+    }
+    else if (ct->ct_flags & CT_ARRAY) {
+        dataoffset = offsetof(CDataObject_own_nolength, alignment);
+        datasize = ct->ct_size;
+        if (datasize < 0) {
+            explicitlength = get_new_array_length(&init);
+            if (explicitlength < 0)
+                return NULL;
+            ctitem = ct->ct_itemdescr;
+            dataoffset = offsetof(CDataObject_own_length, alignment);
+            datasize = explicitlength * ctitem->ct_size;
+            if (explicitlength > 0 &&
+                    (datasize / explicitlength) != ctitem->ct_size) {
+                PyErr_SetString(PyExc_OverflowError,
+                                "array size would overflow a Py_ssize_t");
+                return NULL;
+            }
+        }
+    }
+    else if (ct->ct_flags & CT_PRIMITIVE_ANY) {
+        cd = _new_casted_primitive(ct);
+        datasize = ct->ct_size;
+        goto initialize_casted_primitive;
+    }
+    else {
+        PyErr_Format(PyExc_TypeError,
+                     "cannot create cdata '%s' objects", ct->ct_name);
+        return NULL;
+    }
+
+    cd = allocate_owning_object(dataoffset + datasize, ct);
+    if (cd == NULL)
+        return NULL;
+
+    cd->c_data = ((char *)cd) + dataoffset;
+    if (explicitlength >= 0)
+        ((CDataObject_own_length*)cd)->length = explicitlength;
+
+ initialize_casted_primitive:
+    memset(cd->c_data, 0, datasize);
+    if (init != Py_None) {
+        if (convert_from_object(cd->c_data,
+              (ct->ct_flags & CT_POINTER) ? ct->ct_itemdescr : ct, init) < 0) {
+            Py_DECREF(cd);
+            return NULL;
+        }
+    }
+    return (PyObject *)cd;
+}
+
+static PyObject *ffi_cast(ZefFFIObject *self, PyObject *args)
+{
+    CTypeDescrObject *ct;
+    PyObject *ob, *arg;
+    if (!PyArg_ParseTuple(args, "OO:cast", &arg, &ob))
+        return NULL;
+
+    ct = _ffi_type(self, arg, ACCEPT_STRING|ACCEPT_CTYPE);
+    if (ct == NULL)
+        return NULL;
+
+    return do_cast(ct, ob);
+}
+
+static PyObject *ffi_string(ZefFFIObject *self, PyObject *args)
+{
+    CDataObject *cd;
+    Py_ssize_t maxlen = -1;
+    if (!PyArg_ParseTuple(args, "O!|n:string",
+                          &CData_Type, &cd, &maxlen))
+        return NULL;
+
+    if (cd->c_type->ct_itemdescr != NULL &&
+        cd->c_type->ct_itemdescr->ct_flags & (CT_PRIMITIVE_CHAR |
+                                              CT_PRIMITIVE_SIGNED |
+                                              CT_PRIMITIVE_UNSIGNED)) {
+        Py_ssize_t length = maxlen;
+        if (cd->c_data == NULL) {
+            PyObject *s = cdata_repr(cd);
+            if (s != NULL) {
+                PyErr_Format(PyExc_RuntimeError,
+                             "cannot use string() on %s",
+                             PyText_AS_UTF8(s));
+                Py_DECREF(s);
+            }
+            return NULL;
+        }
+        if (length < 0 && cd->c_type->ct_flags & CT_ARRAY) {
+            length = get_array_length(cd);
+        }
+        if (cd->c_type->ct_itemdescr->ct_size == sizeof(char)) {
+            const char *start = cd->c_data;
+            if (length < 0) {
+                /*READ(start, 1)*/
+                length = strlen(start);
+                /*READ(start, length)*/
+            }
+            else {
+                const char *end;
+                /*READ(start, length)*/
+                end = (const char *)memchr(start, 0, length);
+                if (end != NULL)
+                    length = end - start;
+            }
+            return PyBytes_FromStringAndSize(start, length);
+        }
+    }
+    else if (cd->c_type->ct_flags & CT_IS_ENUM) {
+        abort();
+        //return convert_cdata_to_enum_string(cd, 0);
+    }
+    else if (cd->c_type->ct_flags & CT_IS_BOOL) {
+        /* fall through to TypeError */
+    }
+    else if (cd->c_type->ct_flags & (CT_PRIMITIVE_CHAR |
+                                     CT_PRIMITIVE_SIGNED |
+                                     CT_PRIMITIVE_UNSIGNED)) {
+        /*READ(cd->c_data, cd->c_type->ct_size)*/
+        if (cd->c_type->ct_size == sizeof(char))
+            return PyBytes_FromStringAndSize(cd->c_data, 1);
+    }
+    PyErr_Format(PyExc_TypeError, "string(): unexpected cdata '%s' argument",
+                 cd->c_type->ct_name);
+    return NULL;
+}
+
+static CFieldObject *_ffi_field(CTypeDescrObject *ct, const char *fieldname)
+{
+    CFieldObject *cf;
+    if (ct->ct_stuff == NULL) {
+        PyErr_Format(PyExc_TypeError, "'%s' is incomplete", ct->ct_name);
+        return NULL;
+    }
+    cf = (CFieldObject *)PyDict_GetItemString(ct->ct_stuff, fieldname);
+    if (cf == NULL) {
+        PyErr_Format(PyExc_KeyError, "'%s' has got no field '%s'",
+                     ct->ct_name, fieldname);
+        return NULL;
+    }
+    if (cf->cf_bitshift >= 0) {
+        PyErr_SetString(PyExc_TypeError, "not supported for bitfields");
+        return NULL;
+    }
+    return cf;
+}
+
+static PyObject *ffi_offsetof(ZefFFIObject *self, PyObject *args)
+{
+    PyObject *arg;
+    char *fieldname;
+    CTypeDescrObject *ct;
+    CFieldObject *cf;
+
+    if (!PyArg_ParseTuple(args, "Os:offsetof", &arg, &fieldname))
+        return NULL;
+
+    ct = _ffi_type(self, arg, ACCEPT_STRING|ACCEPT_CTYPE);
+    if (ct == NULL)
+        return NULL;
+
+    if (!(ct->ct_flags & (CT_STRUCT|CT_UNION))) {
+        PyErr_Format(PyExc_TypeError,
+                     "expected a struct or union ctype, got '%s'",
+                     ct->ct_name);
+        return NULL;
+    }
+    cf = _ffi_field(ct, fieldname);
+    if (cf == NULL)
+        return NULL;
+    return PyInt_FromSsize_t(cf->cf_offset);
+}
+
+static PyObject *ffi_addressof(ZefFFIObject *self, PyObject *args)
+{
+    PyObject *obj;
+    char *fieldname = NULL;
+
+    if (!PyArg_ParseTuple(args, "O|z:addressof", &obj, &fieldname))
+        return NULL;
+
+    if (CData_Check(obj)) {
+        CDataObject *cd = (CDataObject *)obj;
+        CTypeDescrObject *ct;
+        Py_ssize_t offset;
+
+        ct = cd->c_type;
+        if (fieldname != NULL && ct->ct_flags & CT_POINTER)
+            ct = ct->ct_itemdescr;
+
+        if (!(ct->ct_flags & (CT_STRUCT|CT_UNION))) {
+            PyErr_Format(PyExc_TypeError,
+                         "expected a struct or union cdata, got '%s'",
+                         ct->ct_name);
+            return NULL;
+        }
+
+        if (fieldname == NULL) {
+            offset = 0;
+        }
+        else {
+            CFieldObject *cf = _ffi_field(ct, fieldname);
+            if (cf == NULL)
+                return NULL;
+            offset = cf->cf_offset;
+            ct = cf->cf_type;
+        }
+        ct = fetch_pointer_type(self->types_dict, ct);
+        if (ct == NULL)
+            return NULL;
+        return new_simple_cdata(cd->c_data + offset, ct);
+    }
+    else if (ZefLib_Check(obj)) {
+        PyObject *attr, *name;
+        char *reason;
+
+        if (fieldname == NULL) {
+            PyErr_SetString(PyExc_TypeError, "addressof(Lib, fieldname) "
+                            "cannot be used with only one argument");
+            return NULL;
+        }
+        name = PyString_FromString(fieldname);
+        if (name == NULL)
+            return NULL;
+        attr = lib_findattr((ZefLibObject *)obj, name, ZefError);
+        Py_DECREF(name);
+        if (attr == NULL)
+            return NULL;
+
+        if (ZefGlobSupport_Check(attr)) {
+            return addressof_global_var((ZefGlobSupportObject *)attr);
+        }
+
+        if (PyCFunction_Check(attr))
+            reason = "declare that function as a function pointer instead";
+        else
+            reason = "numeric constants don't have addresses";
+
+        PyErr_Format(PyExc_TypeError,
+                     "cannot take the address of '%s' (%s)",
+                     fieldname, reason);
+        return NULL;
+    }
+    else {
+        PyErr_SetString(PyExc_TypeError, "addressof() first argument must be "
+                        "a cdata struct or union, a pointer to one, or a Lib "
+                        "object");
+        return NULL;
+    }
+}
+
+static PyObject *ffi_getctype(ZefFFIObject *self, PyObject *args)
+{
+    PyObject *cdecl, *res;
+    char *p, *replace_with = "";
+    int add_paren, add_space;
+    CTypeDescrObject *ct;
+    size_t replace_with_len;
+
+    if (!PyArg_ParseTuple(args, "O|s:getctype", &cdecl, &replace_with))
+        return NULL;
+
+    ct = _ffi_type(self, cdecl, ACCEPT_STRING|ACCEPT_CTYPE);
+    if (ct == NULL)
+        return NULL;
+
+    while (replace_with[0] != 0 && isspace(replace_with[0]))
+        replace_with++;
+    replace_with_len = strlen(replace_with);
+    while (replace_with_len > 0 && isspace(replace_with[replace_with_len - 1]))
+        replace_with_len--;
+
+    add_paren = (replace_with[0] == '*' &&
+                 ((ct->ct_flags & (CT_ARRAY | CT_FUNCTION)) != 0));
+    add_space = (!add_paren && replace_with_len > 0 &&
+                 replace_with[0] != '[' && replace_with[0] != '(');
+
+    res = combine_type_name_l(ct, replace_with_len + add_space + 2 * add_paren);
+    if (res == NULL)
+        return NULL;
+
+    p = PyString_AS_STRING(res) + ct->ct_name_position;
+    if (add_paren)
+        *p++ = '(';
+    if (add_space)
+        *p++ = ' ';
+    memcpy(p, replace_with, replace_with_len);
+    if (add_paren)
+        p[replace_with_len] = ')';
+    return res;
+}
+
+static PyObject *ffi_new_handle(ZefFFIObject *self, PyObject *arg)
+{
+    CTypeDescrObject *ct = ZefNULL->c_type;   // 
+    CDataObject *cd;
+
+    cd = (CDataObject *)PyObject_GC_New(CDataObject, &CDataOwningGC_Type);
+    if (cd == NULL)
+        return NULL;
+    Py_INCREF(ct);
+    cd->c_type = ct;
+    Py_INCREF(arg);
+    cd->c_data = ((char *)arg) - 42;
+    cd->c_weakreflist = NULL;
+    PyObject_GC_Track(cd);
+    return (PyObject *)cd;
+}
+
+static PyObject *ffi_from_handle(PyObject *self, PyObject *arg)
+{
+    CTypeDescrObject *ct;
+    char *raw;
+    PyObject *x;
+    if (!CData_Check(arg)) {
+        PyErr_SetString(PyExc_TypeError, "expected a 'cdata' object");
+        return NULL;
+    }
+    ct = ((CDataObject *)arg)->c_type;
+    raw = ((CDataObject *)arg)->c_data;
+    if (!(ct->ct_flags & CT_CAST_ANYTHING)) {
+        PyErr_Format(PyExc_TypeError,
+                     "expected a 'cdata' object with a 'void *' out of "
+                     "new_handle(), got '%s'", ct->ct_name);
+        return NULL;
+    }
+    if (!raw) {
+        PyErr_SetString(PyExc_RuntimeError,
+                        "cannot use from_handle() on NULL pointer");
+        return NULL;
+    }
+    x = (PyObject *)(raw + 42);
+    Py_INCREF(x);
+    return x;
+}
+
+static PyObject *ffi_gc(ZefFFIObject *self, PyObject *args)
+{
+    CDataObject *cd;
+    PyObject *destructor;
+
+    if (!PyArg_ParseTuple(args, "O!O:gc", &CData_Type, &cd, &destructor))
+        return NULL;
+
+    return gc_weakrefs_build(self, cd, destructor);
+}
+#endif
+
+static PyMethodDef ffi_methods[] = {
+#if 0
+    {"addressof",     (PyCFunction)ffi_addressof, METH_VARARGS},
+    {"cast",          (PyCFunction)ffi_cast,      METH_VARARGS},
+    {"close_library", ffi_close_library,          METH_VARARGS | METH_STATIC},
+    {"from_handle",   (PyCFunction)ffi_from_handle,METH_O},
+    {"gc",            (PyCFunction)ffi_gc,        METH_VARARGS},
+    {"getctype",      (PyCFunction)ffi_getctype,  METH_VARARGS},
+    {"load_library",  (PyCFunction)ffi_load_library,METH_VARARGS|METH_KEYWORDS},
+    {"offsetof",      (PyCFunction)ffi_offsetof,  METH_VARARGS},
+    {"new",           (PyCFunction)ffi_new,       METH_VARARGS},
+    {"new_handle",    (PyCFunction)ffi_new_handle,METH_O},
+#endif
+    {"sizeof",        (PyCFunction)ffi_sizeof,    METH_O},
+#if 0
+    {"string",        (PyCFunction)ffi_string,    METH_VARARGS},
+    {"typeof",        (PyCFunction)ffi_typeof,    METH_O},
+#endif
+    {NULL}
+};
+
+static PyTypeObject FFI_Type = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "cffi.FFI",
+    sizeof(FFIObject),
+    0,
+    (destructor)ffi_dealloc,                    /* tp_dealloc */
+    0,                                          /* tp_print */
+    0,                                          /* tp_getattr */
+    0,                                          /* tp_setattr */
+    0,                                          /* tp_compare */
+    0,                                          /* tp_repr */
+    0,                                          /* tp_as_number */
+    0,                                          /* tp_as_sequence */
+    0,                                          /* tp_as_mapping */
+    0,                                          /* tp_hash */
+    0,                                          /* tp_call */
+    0,                                          /* tp_str */
+    PyObject_GenericGetAttr,                    /* tp_getattro */
+    0,                                          /* tp_setattro */
+    0,                                          /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,    /* tp_flags */
+    0,                                          /* tp_doc */
+    (traverseproc)ffi_traverse,                 /* tp_traverse */
+    0,                                          /* tp_clear */
+    0,                                          /* tp_richcompare */
+    0,                                          /* tp_weaklistoffset */
+    0,                                          /* tp_iter */
+    0,                                          /* tp_iternext */
+    ffi_methods,                                /* tp_methods */
+    0,                                          /* tp_members */
+    0,                                          /* tp_getset */
+    0,                                          /* tp_base */
+    0,                                          /* tp_dict */
+    0,                                          /* tp_descr_get */
+    0,                                          /* tp_descr_set */
+    0,                                          /* tp_dictoffset */
+    0,                                          /* tp_init */
+    0,                                          /* tp_alloc */
+    ffiobj_new,                                 /* tp_new */
+    PyObject_GC_Del,                            /* tp_free */
+};
diff --git a/new/lib_obj.c b/new/lib_obj.c
new file mode 100644
--- /dev/null
+++ b/new/lib_obj.c
@@ -0,0 +1,197 @@
+
+/* A Lib object is what is returned by any of:
+
+   - the "lib" attribute of a C extension module originally created by
+     recompile()
+
+   - ffi.dlopen()
+
+   - ffi.verify()
+
+   A Lib object is special in the sense that it has a custom
+   __getattr__ which returns C globals, functions and constants.  It
+   raises AttributeError for anything else, like '__class__'.
+
+   A Lib object has internally a reference back to the FFI object,
+   which holds the _cffi_type_context_s used to create lazily the
+   objects returned by __getattr__.  For a dlopen()ed Lib object, all
+   the 'address' fields in _cffi_global_s are NULL, and instead
+   dlsym() is used lazily on the l_dl_lib.
+*/
+
+struct LibObject_s {
+    PyObject_HEAD
+    PyObject *l_dict;           /* content, built lazily */
+    struct FFIObject_s *l_ffi;  /* ffi object */
+    void *l_dl_lib;             /* the result of 'dlopen()', or NULL */
+};
+
+#define ZefLib_Check(ob)  ((Py_TYPE(ob) == &ZefLib_Type))
+
+static void lib_dealloc(ZefLibObject *lib)
+{
+    (void)lib_close(lib);
+    PyObject_Del(lib);
+}
+
+static PyObject *lib_repr(ZefLibObject *lib)
+{
+    return PyText_FromFormat("",
+                             lib->l_libname,
+                             lib->l_dl_lib == NULL ? " (closed)" : "");
+}
+
+static PyObject *lib_findattr(ZefLibObject *lib, PyObject *name, PyObject *exc)
+{
+    /* does not return a new reference! */
+
+    if (lib->l_dict == NULL) {
+        PyErr_Format(ZefError, "lib '%.200s' was closed", lib->l_libname);
+        return NULL;
+    }
+
+    PyObject *x = PyDict_GetItem(lib->l_dict, name);
+    if (x == NULL) {
+        PyErr_Format(exc,
+                     "lib '%.200s' has no function,"
+                     " global variable or constant '%.200s'",
+                     lib->l_libname,
+                     PyText_Check(name) ? PyText_AS_UTF8(name) : "?");
+        return NULL;
+    }
+    return x;
+}
+
+static PyObject *lib_getattr(ZefLibObject *lib, PyObject *name)
+{
+    PyObject *x = lib_findattr(lib, name, PyExc_AttributeError);
+    if (x == NULL)
+        return NULL;
+
+    if (ZefGlobSupport_Check(x)) {
+        return read_global_var((ZefGlobSupportObject *)x);
+    }
+    Py_INCREF(x);
+    return x;
+}
+
+static int lib_setattr(ZefLibObject *lib, PyObject *name, PyObject *val)
+{
+    PyObject *x = lib_findattr(lib, name, PyExc_AttributeError);
+    if (x == NULL)
+        return -1;
+
+    if (val == NULL) {
+        PyErr_SetString(PyExc_AttributeError,
+                        "cannot delete attributes from Lib object");
+        return -1;
+    }
+
+    if (ZefGlobSupport_Check(x)) {
+        return write_global_var((ZefGlobSupportObject *)x, val);
+    }
+
+    PyErr_Format(PyExc_AttributeError,
+                 "cannot write to function or constant '%.200s'",
+                 PyText_Check(name) ? PyText_AS_UTF8(name) : "?");
+    return -1;
+}
+
+static PyObject *lib_dir(PyObject *lib, PyObject *noarg)
+{
+    return PyDict_Keys(((ZefLibObject *)lib)->l_dict);
+}
+
+static PyMethodDef lib_methods[] = {
+    {"__dir__",   lib_dir,  METH_NOARGS},
+    {NULL,        NULL}           /* sentinel */
+};
+
+static PyTypeObject ZefLib_Type = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "zeffir.Lib",
+    sizeof(ZefLibObject),
+    0,
+    (destructor)lib_dealloc,                    /* tp_dealloc */
+    0,                                          /* tp_print */
+    0,                                          /* tp_getattr */
+    0,                                          /* tp_setattr */
+    0,                                          /* tp_compare */
+    (reprfunc)lib_repr,                         /* tp_repr */
+    0,                                          /* tp_as_number */
+    0,                                          /* tp_as_sequence */
+    0,                                          /* tp_as_mapping */
+    0,                                          /* tp_hash */
+    0,                                          /* tp_call */
+    0,                                          /* tp_str */
+    (getattrofunc)lib_getattr,                  /* tp_getattro */
+    (setattrofunc)lib_setattr,                  /* tp_setattro */
+    0,                                          /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT,                         /* tp_flags */
+    0,                                          /* tp_doc */
+    0,                                          /* tp_traverse */
+    0,                                          /* tp_clear */
+    0,                                          /* tp_richcompare */
+    0,                                          /* tp_weaklistoffset */
+    0,                                          /* tp_iter */
+    0,                                          /* tp_iternext */
+    lib_methods,                                /* tp_methods */
+    0,                                          /* tp_members */
+    0,                                          /* tp_getset */
+    0,                                          /* tp_base */
+    0,                                          /* tp_dict */
+    0,                                          /* tp_descr_get */
+    0,                                          /* tp_descr_set */
+    offsetof(ZefLibObject, l_dict),             /* tp_dictoffset */
+};
+
+static void lib_dlerror(ZefLibObject *lib)
+{
+    char *error = dlerror();
+    if (error == NULL)
+        error = "(no error reported)";
+    PyErr_Format(PyExc_OSError, "%s: %s", lib->l_libname, error);
+}
+
+static ZefLibObject *lib_create(PyObject *path)
+{
+    ZefLibObject *lib;
+
+    lib = PyObject_New(ZefLibObject, &ZefLib_Type);
+    if (lib == NULL)
+        return NULL;
+
+    lib->l_dl_lib = NULL;
+    lib->l_libname = PyString_AsString(path);
+    Py_INCREF(path);
+    lib->l_libname_obj = path;
+    lib->l_dict = PyDict_New();
+    if (lib->l_dict == NULL) {
+        Py_DECREF(lib);
+        return NULL;
+    }
+
+    lib->l_dl_lib = dlopen(lib->l_libname, RTLD_LAZY);
+    if (lib->l_dl_lib == NULL) {
+        lib_dlerror(lib);
+        Py_DECREF(lib);
+        return NULL;
+    }
+    return lib;
+}
+
+static int lib_close(ZefLibObject *lib)
+{
+    void *dl_lib;
+    Py_CLEAR(lib->l_dict);
+
+    dl_lib = lib->l_dl_lib;
+    if (dl_lib != NULL) {
+        lib->l_dl_lib = NULL;
+        if (dlclose(dl_lib) != 0) {
+            lib_dlerror(lib);
+            return -1;
+        }
+    }
+    return 0;
+}
diff --git a/new/manual.c b/new/manual.c
new file mode 100644
--- /dev/null
+++ b/new/manual.c
@@ -0,0 +1,109 @@
+#include "_cffi_include.h"
+
+
+int foo42(int a, int *b)
+{
+    return a - *b;
+}
+
+int foo64(int a)
+{
+    return ~a;
+}
+
+/************************************************************/
+
+static void *_cffi_types[] = {
+    _CFFI_OP(_CFFI_OP_FUNCTION, 1),
+    _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_INT),
+    _CFFI_OP(_CFFI_OP_POINTER, 1),
+    _CFFI_OP(_CFFI_OP_FUNCTION_END, 0),
+    _CFFI_OP(_CFFI_OP_FUNCTION, 1),
+    _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_INT),
+    _CFFI_OP(_CFFI_OP_FUNCTION_END, 0),
+};
+
+static PyObject *
+_cffi_f_foo42(PyObject *self, PyObject *args)
+{
+  int x0;
+  int * x1;
+  Py_ssize_t datasize;
+  int result;
+  PyObject *arg0;
+  PyObject *arg1;
+
+  if (!PyArg_ParseTuple(args, "OO:foo42", &arg0, &arg1))
+    return NULL;
+
+  x0 = _cffi_to_c_int(arg0, int);
+  if (x0 == (int)-1 && PyErr_Occurred())
+    return NULL;
+
+  datasize = _cffi_prepare_pointer_call_argument(
+      _cffi_types[1], arg1, (char **)&x1);
+  if (datasize != 0) {
+    if (datasize < 0)
+      return NULL;
+    x1 = alloca(datasize);
+    memset((void *)x1, 0, datasize);
+    if (_cffi_convert_array_from_object((char *)x1, _cffi_types[1], arg1) < 0)
+      return NULL;
+  }
+
+  Py_BEGIN_ALLOW_THREADS
+  _cffi_restore_errno();
+  { result = foo42(x0, x1); }
+  _cffi_save_errno();
+  Py_END_ALLOW_THREADS
+
+  return _cffi_from_c_int(result, int);
+}
+
+static PyObject *
+_cffi_f_foo64(PyObject *self, PyObject *arg0)
+{
+  int x0;
+  int result;
+
+  x0 = _cffi_to_c_int(arg0, int);
+  if (x0 == (int)-1 && PyErr_Occurred())
+    return NULL;
+
+  Py_BEGIN_ALLOW_THREADS
+  _cffi_restore_errno();
+  { result = foo64(x0); }
+  _cffi_save_errno();
+  Py_END_ALLOW_THREADS
+
+  return _cffi_from_c_int(result, int);
+}
+
+static const struct _cffi_global_s _cffi_globals[] = {
+    { "foo42", &_cffi_f_foo42, _CFFI_OP(_CFFI_OP_CPYTHON_BLTN, 0) },
+    { "foo64", &_cffi_f_foo64, _CFFI_OP(_CFFI_OP_CPYTHON_BLTN, 4) },
+};
+
+static const struct _cffi_type_context_s _cffi_type_context = {
+    _cffi_types,
+    _cffi_globals,
+    NULL,  /* no constants */
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    1,  /* num_globals */
+    0,
+    0,
+    0,
+    0,
+};
+
+PyMODINIT_FUNC
+initmanual(void)
+{
+    if (_cffi_init() < 0)
+        return;
+
+    _cffi_init_module("manual", &_cffi_type_context);
+}
diff --git a/new/parse_c_type.c b/new/parse_c_type.c
--- a/new/parse_c_type.c
+++ b/new/parse_c_type.c
@@ -41,9 +41,9 @@
 
 typedef struct {
     struct _cffi_parse_info_s *info;
-    const char *p;
+    const char *input, *p;
+    size_t size;              // the next token is at 'p' and of length 'size'
     enum token_e kind;
-    size_t size;
     _cffi_opcode_t *output;
     size_t output_index;
 } token_t;
@@ -177,7 +177,7 @@
 {
     if (tok->kind != TOK_ERROR) {
         tok->kind = TOK_ERROR;
-        tok->info->error_location = tok->p;
+        tok->info->error_location = tok->p - tok->input;
         tok->info->error_message = msg;
     }
     return -1;
@@ -349,7 +349,7 @@
     return _CFFI_GETARG(result);
 }
 
-static int search_struct_union(struct _cffi_type_context_s *ctx,
+static int search_struct_union(const struct _cffi_type_context_s *ctx,
                                const char *search, size_t search_len)
 {
     int left = 0, right = ctx->num_structs_unions;
@@ -368,7 +368,7 @@
     return -1;
 }
 
-static int search_typename(struct _cffi_type_context_s *ctx,
+static int search_typename(const struct _cffi_type_context_s *ctx,
                            const char *search, size_t search_len)
 {
     int left = 0, right = ctx->num_typenames;
@@ -554,6 +554,7 @@
 
     token.info = info;
     token.kind = TOK_START;
+    token.input = input;
     token.p = input;
     token.size = 0;
     token.output = info->output;
diff --git a/new/parse_c_type.h b/new/parse_c_type.h
--- a/new/parse_c_type.h
+++ b/new/parse_c_type.h
@@ -95,10 +95,10 @@
 };
 
 struct _cffi_parse_info_s {
-    struct _cffi_type_context_s *ctx;
+    const struct _cffi_type_context_s *ctx;
     _cffi_opcode_t *output;
     int output_size;
-    const char *error_location;
+    size_t error_location;
     const char *error_message;
 };
 
diff --git a/new/realize_c_type.c b/new/realize_c_type.c
--- a/new/realize_c_type.c
+++ b/new/realize_c_type.c
@@ -1,22 +1,19 @@
-#include 
-#include "parse_c_type.h"
 
+static CTypeDescrObject *all_primitives[_CFFI__NUM_PRIM];
 
-static PyObject *all_primitives[_CFFI__NUM_PRIM];
 
-
-PyObject *build_primitive_type(int num)
+CTypeDescrObject *build_primitive_type(int num)
 {
-    PyObject *x;
+    CTypeDescrObject *x;
 
     switch (num) {
 
     case _CFFI_PRIM_VOID:
-        x = PyString_FromString("VOID");
+        x = new_void_type();
         break;
 
     case _CFFI_PRIM_INT:
-        x = PyString_FromString("INT");
+        x = new_primitive_type("int");
         break;
 
     default:
@@ -29,15 +26,21 @@
 }
 
 
-PyObject *realize_c_type(struct _cffi_type_context_s *ctx,
-                         _cffi_opcode_t opcodes[], int index)
+/* Interpret an opcodes[] array.  If opcodes == ctx->types, store all
+   the intermediate types back in the opcodes[].  Returns a new
+   reference.
+*/
+CTypeDescrObject *realize_c_type(const struct _cffi_type_context_s *ctx,
+                                 _cffi_opcode_t opcodes[], int index)
 {
-    PyObject *x, *y;
+    CTypeDescrObject *ct;
+    CTypeDescrObject *x, *y;
     _cffi_opcode_t op = opcodes[index];
+
     if ((((uintptr_t)op) & 1) == 0) {
-        x = (PyObject *)op;
-        Py_INCREF(x);
-        return x;
+        ct = (CTypeDescrObject *)op;
+        Py_INCREF(ct);
+        return ct;
     }
 
     switch (_CFFI_GETOP(op)) {
@@ -53,7 +56,7 @@
         y = realize_c_type(ctx, opcodes, _CFFI_GETARG(op));
         if (y == NULL)
             return NULL;
-        x = Py_BuildValue("sO", "pointer", y);
+        x = new_pointer_type(y);
         Py_DECREF(y);
         break;
 
@@ -62,44 +65,10 @@
         return NULL;
     }
 
-    if (opcodes == ctx->types) {
+    if (x != NULL && opcodes == ctx->types) {
+        assert((((uintptr_t)x) & 1) == 0);
         Py_INCREF(x);
         opcodes[index] = x;
     }
     return x;
 };
-
-
-static PyObject *b_test(PyObject *self, PyObject *args)
-{
-    char *s;
-    if (!PyArg_ParseTuple(args, "s", &s))
-        return NULL;
-
-    _cffi_opcode_t opcodes[100];
-    struct _cffi_type_context_s global_ctx = { NULL };
-    struct _cffi_parse_info_s parse_info = {
-        .ctx = &global_ctx,
-        .output = opcodes,
-        .output_size = 100,
-    };
-    int res = parse_c_type(&parse_info, s);
-    if (res < 0) {
-        PyErr_SetString(PyExc_ValueError, parse_info.error_message);
-        return NULL;
-    }
-
-    return realize_c_type(&global_ctx, opcodes, res);
-}
-
-static PyMethodDef MyMethods[] = {
-    {"test",   b_test,  METH_VARARGS},
-    {NULL,     NULL}    /* Sentinel */
-};
-
-PyMODINIT_FUNC
-initrealize_c_type(void)
-{
-    PyObject *m = Py_InitModule("realize_c_type", MyMethods);
-    (void)m;
-}
diff --git a/new/setup_manual.py b/new/setup_manual.py
new file mode 100644
--- /dev/null
+++ b/new/setup_manual.py
@@ -0,0 +1,5 @@
+from distutils.core import setup
+from distutils.extension import Extension
+setup(name='manual',
+      ext_modules=[Extension(name='manual',
+                             sources=['manual.c'])])
diff --git a/new/test_parse_c_type.py b/new/test_parse_c_type.py
--- a/new/test_parse_c_type.py
+++ b/new/test_parse_c_type.py
@@ -49,11 +49,10 @@
     info.output_size = len(out)
     for j in range(len(out)):
         out[j] = ffi.cast("void *", -424242)
-    c_input = ffi.new("char[]", input)
-    res = lib.parse_c_type(info, c_input)
+    res = lib.parse_c_type(info, input)
     if res < 0:
         raise ParseError(ffi.string(info.error_message),
-                         info.error_location - c_input)
+                         info.error_location)
     assert 0 <= res < len(out)
     result = []
     for j in range(len(out)):

From noreply at buildbot.pypy.org  Tue Apr 14 15:54:34 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 14 Apr 2015 15:54:34 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Rename "_cffi_backend.so" to
 "_cffi1_backend.so" to avoid confusion with
Message-ID: <20150414135434.C06BA1C0661@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1700:5694f0b6aee0
Date: 2015-04-14 15:54 +0200
http://bitbucket.org/cffi/cffi/changeset/5694f0b6aee0/

Log:	Rename "_cffi_backend.so" to "_cffi1_backend.so" to avoid confusion
	with old versions of cffi.

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -5272,12 +5272,6 @@
     return NULL;
 }
 
-static PyObject *b__get_types(PyObject *self, PyObject *noarg)
-{
-    return PyTuple_Pack(2, (PyObject *)&CData_Type,
-                           (PyObject *)&CTypeDescr_Type);
-}
-
 /************************************************************/
 
 static char _testfunc0(char a, char b)
@@ -5582,7 +5576,6 @@
 #ifdef MS_WIN32
     {"getwinerror", b_getwinerror, METH_VARARGS},
 #endif
-    {"_get_types", b__get_types, METH_NOARGS},
     {"_testfunc", b__testfunc, METH_VARARGS},
     {"_testbuff", b__testbuff, METH_VARARGS},
     {NULL,     NULL}    /* Sentinel */
@@ -5738,7 +5731,7 @@
 #if PY_MAJOR_VERSION >= 3
 static struct PyModuleDef FFIBackendModuleDef = {
   PyModuleDef_HEAD_INIT,
-  "_cffi_backend",
+  "_cffi1_backend",
   NULL,
   -1,
   FFIBackendMethods,
@@ -5752,7 +5745,7 @@
 #define INITERROR return
 
 PyMODINIT_FUNC
-init_cffi_backend(void)
+init_cffi1_backend(void)
 #endif
 {
     PyObject *m, *v;
@@ -5769,7 +5762,7 @@
 #if PY_MAJOR_VERSION >= 3
     m = PyModule_Create(&FFIBackendModuleDef);
 #else
-    m = Py_InitModule("_cffi_backend", FFIBackendMethods);
+    m = Py_InitModule("_cffi1_backend", FFIBackendMethods);
 #endif
 
     if (m == NULL)
@@ -5848,6 +5841,9 @@
 
     init_errno();
 
+    if (init_ffi_lib(m) < 0)
+        INITERROR;
+
 #if PY_MAJOR_VERSION >= 3
     if (init_file_emulator() < 0)
         INITERROR;
diff --git a/new/_cffi_include.h b/new/_cffi_include.h
--- a/new/_cffi_include.h
+++ b/new/_cffi_include.h
@@ -161,7 +161,7 @@
     PyObject *module, *c_api_object = NULL;
     void *src;
 
-    module = PyImport_ImportModule("_cffi_backend");
+    module = PyImport_ImportModule("_cffi1_backend");
     if (module == NULL)
         goto failure;
 
@@ -175,7 +175,7 @@
     src = PyCapsule_GetPointer(c_api_object, "cffi");
     if ((uintptr_t)(((void **)src)[0]) < _CFFI_NUM_EXPORTS) {
         PyErr_SetString(PyExc_ImportError,
-                        "the _cffi_backend module is an outdated version");
+                        "the _cffi1_backend module is an outdated version");
         goto failure;
     }
     memcpy(_cffi_exports, src, _CFFI_NUM_EXPORTS * sizeof(void *));
diff --git a/new/cffi1_module.c b/new/cffi1_module.c
--- a/new/cffi1_module.c
+++ b/new/cffi1_module.c
@@ -13,12 +13,8 @@
 //#include "lib_obj.c"
 
 
-static int init_ffi_lib(void)
+static int init_ffi_lib(PyObject *m)
 {
-    static int ffi_lib_ok = 0;
-    if (ffi_lib_ok)
-        return 0;
-
     if (!PyType_Ready(&FFI_Type) < 0)
         return -1;
     if (!PyType_Ready(&Lib_Type) < 0)
@@ -30,16 +26,16 @@
     if (PyDict_SetItemString(FFI_Type.tp_dict, "error", FFIError) < 0)
         return -1;
 
-    ffi_lib_ok = 1;
+    Py_INCREF(&FFI_Type);
+    if (PyModule_AddObject(m, "FFI", (PyObject *)&FFI_Type) < 0)
+        return -1;
+
     return 0;
 }
 
 static int _cffi_init_module(char *module_name,
                              const struct _cffi_type_context_s *ctx)
 {
-    if (init_ffi_lib() < 0)
-        return -1;
-
     PyObject *m = Py_InitModule(module_name, NULL);
     if (m == NULL)
         return -1;
diff --git a/new/ffi_obj.c b/new/ffi_obj.c
--- a/new/ffi_obj.c
+++ b/new/ffi_obj.c
@@ -65,9 +65,13 @@
 static PyObject *ffiobj_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 {
     /* user-facing initialization code, for explicit FFI() calls */
-    /* XXX LATER */
-    PyErr_SetString(PyExc_NotImplementedError, "FFI()");
-    return NULL;
+    static const struct _cffi_type_context_s empty_ctx = { 0 };
+
+    char *keywords[] = {NULL};
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, ":FFI", keywords))
+        return NULL;
+
+    return (PyObject *)ffi_internal_new(&empty_ctx);
 }
 
 #define ACCEPT_STRING   1
diff --git a/new/test_realize_c_type.py b/new/test_realize_c_type.py
--- a/new/test_realize_c_type.py
+++ b/new/test_realize_c_type.py
@@ -1,15 +1,16 @@
-import os
 
-def setup_module():
-    err = os.system("python setup.py build_ext -i")
-    if err != 0:
-        raise Exception(err)
-    global realize_c_type
-    import realize_c_type
 
+def check(input, expected_output=None):
+    import _cffi1_backend
+    ffi = _cffi1_backend.FFI()
+    ct = ffi.typeof(input)
+    assert isinstance(ct, ffi.CType)
+    assert ct.cname == (expected_output or input)
 
 def test_void():
-    assert realize_c_type.test("void") == "VOID"
+    check("void", "void")
+    check("  void  ", "void")
 
 def test_int_star():
-    assert realize_c_type.test("int *") == ("pointer", "INT")
+    check("int")
+    check("int *")
diff --git a/setup_base.py b/setup_base.py
--- a/setup_base.py
+++ b/setup_base.py
@@ -9,9 +9,7 @@
     from distutils.core import setup
     from distutils.extension import Extension
     standard = '__pypy__' not in sys.modules
-    setup(packages=['cffi'],
-          requires=['pycparser'],
-          ext_modules=[Extension(name = '_cffi_backend',
+    setup(ext_modules=[Extension(name = '_cffi1_backend',
                                  include_dirs=include_dirs,
                                  sources=sources,
                                  libraries=libraries,

From noreply at buildbot.pypy.org  Tue Apr 14 15:57:36 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 14 Apr 2015 15:57:36 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Pass test_realize_c_type
Message-ID: <20150414135736.C028F1C0698@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1701:ac35ddf71ece
Date: 2015-04-14 15:58 +0200
http://bitbucket.org/cffi/cffi/changeset/ac35ddf71ece/

Log:	Pass test_realize_c_type

diff --git a/new/cffi1_module.c b/new/cffi1_module.c
--- a/new/cffi1_module.c
+++ b/new/cffi1_module.c
@@ -25,6 +25,9 @@
         return -1;
     if (PyDict_SetItemString(FFI_Type.tp_dict, "error", FFIError) < 0)
         return -1;
+    if (PyDict_SetItemString(FFI_Type.tp_dict, "CType",
+                             (PyObject *)&CTypeDescr_Type) < 0)
+        return -1;
 
     Py_INCREF(&FFI_Type);
     if (PyModule_AddObject(m, "FFI", (PyObject *)&FFI_Type) < 0)
diff --git a/new/ffi_obj.c b/new/ffi_obj.c
--- a/new/ffi_obj.c
+++ b/new/ffi_obj.c
@@ -148,21 +148,19 @@
     return PyInt_FromSsize_t(ct->ct_size);
 }
 
-#if 0
-static PyObject *ffi_typeof(ZefFFIObject *self, PyObject *arg)
+static PyObject *ffi_typeof(FFIObject *self, PyObject *arg)
 {
     PyObject *x = (PyObject *)_ffi_type(self, arg, ACCEPT_STRING|ACCEPT_CDATA);
     if (x != NULL) {
         Py_INCREF(x);
     }
     else if (PyCFunction_Check(arg)) {
-        PyErr_SetString(PyExc_TypeError, "typeof(lib.func) not supported: the "
-                        "exact type of functions is unknown (declare that "
-                        "function as a function pointer instead)");
+        abort(); // XXX
     }
     return x;
 }
 
+#if 0
 static PyObject *ffi_new(ZefFFIObject *self, PyObject *args)
 {
     CTypeDescrObject *ct, *ctitem;
@@ -559,8 +557,8 @@
     {"sizeof",        (PyCFunction)ffi_sizeof,    METH_O},
 #if 0
     {"string",        (PyCFunction)ffi_string,    METH_VARARGS},
+#endif
     {"typeof",        (PyCFunction)ffi_typeof,    METH_O},
-#endif
     {NULL}
 };
 

From noreply at buildbot.pypy.org  Tue Apr 14 16:13:04 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 14 Apr 2015 16:13:04 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Arrays
Message-ID: <20150414141304.418801C06CA@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1702:802ad7fc65d4
Date: 2015-04-14 16:13 +0200
http://bitbucket.org/cffi/cffi/changeset/802ad7fc65d4/

Log:	Arrays

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -1902,8 +1902,8 @@
     return cd->c_data + i * cd->c_type->ct_itemdescr->ct_size;
 }
 
-static PyObject *
-new_array_type(CTypeDescrObject *ctptr, PyObject *lengthobj);   /* forward */
+static CTypeDescrObject *
+new_array_type(CTypeDescrObject *ctptr, Py_ssize_t length);   /* forward */
 
 static CTypeDescrObject *
 _cdata_getslicearg(CDataObject *cd, PySliceObject *slice, Py_ssize_t bounds[])
@@ -1968,7 +1968,7 @@
         return NULL;
 
     if (ct->ct_stuff == NULL) {
-        ct->ct_stuff = new_array_type(ct, Py_None);
+        ct->ct_stuff = (PyObject *)new_array_type(ct, -1);
         if (ct->ct_stuff == NULL)
             return NULL;
     }
@@ -3540,41 +3540,15 @@
 static PyObject *b_new_array_type(PyObject *self, PyObject *args)
 {
     PyObject *lengthobj;
+    Py_ssize_t length;
     CTypeDescrObject *ctptr;
 
     if (!PyArg_ParseTuple(args, "O!O:new_array_type",
                           &CTypeDescr_Type, &ctptr, &lengthobj))
         return NULL;
 
-    return new_array_type(ctptr, lengthobj);
-}
-
-static PyObject *
-new_array_type(CTypeDescrObject *ctptr, PyObject *lengthobj)
-{
-    CTypeDescrObject *td, *ctitem;
-    char extra_text[32];
-    Py_ssize_t length, arraysize;
-    int flags = CT_ARRAY;
-
-    if (!(ctptr->ct_flags & CT_POINTER)) {
-        PyErr_SetString(PyExc_TypeError, "first arg must be a pointer ctype");
-        return NULL;
-    }
-    ctitem = ctptr->ct_itemdescr;
-    if (ctitem->ct_size < 0) {
-        PyErr_Format(PyExc_ValueError, "array item of unknown size: '%s'",
-                     ctitem->ct_name);
-        return NULL;
-    }
-
     if (lengthobj == Py_None) {
-        sprintf(extra_text, "[]");
         length = -1;
-        arraysize = -1;
-        if ((ctitem->ct_flags & CT_PRIMITIVE_CHAR) &&
-                ctitem->ct_size == sizeof(char))
-            flags |= CT_IS_UNSIZED_CHAR_A;
     }
     else {
         length = PyNumber_AsSsize_t(lengthobj, PyExc_OverflowError);
@@ -3583,6 +3557,38 @@
                 PyErr_SetString(PyExc_ValueError, "negative array length");
             return NULL;
         }
+    }
+    return (PyObject *)new_array_type(ctptr, length);
+}
+
+static CTypeDescrObject *
+new_array_type(CTypeDescrObject *ctptr, Py_ssize_t length)
+{
+    CTypeDescrObject *td, *ctitem;
+    char extra_text[32];
+    Py_ssize_t arraysize;
+    int flags = CT_ARRAY;
+
+    if (!(ctptr->ct_flags & CT_POINTER)) {
+        PyErr_SetString(PyExc_TypeError, "first arg must be a pointer ctype");
+        return NULL;
+    }
+    ctitem = ctptr->ct_itemdescr;
+    if (ctitem->ct_size < 0) {
+        PyErr_Format(PyExc_ValueError, "array item of unknown size: '%s'",
+                     ctitem->ct_name);
+        return NULL;
+    }
+
+    if (length < 0) {
+        sprintf(extra_text, "[]");
+        length = -1;
+        arraysize = -1;
+        if ((ctitem->ct_flags & CT_PRIMITIVE_CHAR) &&
+                ctitem->ct_size == sizeof(char))
+            flags |= CT_IS_UNSIZED_CHAR_A;
+    }
+    else {
         sprintf(extra_text, "[%llu]", (unsigned PY_LONG_LONG)length);
         arraysize = length * ctitem->ct_size;
         if (length > 0 && (arraysize / length) != ctitem->ct_size) {
@@ -3600,7 +3606,7 @@
     td->ct_size = arraysize;
     td->ct_length = length;
     td->ct_flags = flags;
-    return (PyObject *)td;
+    return td;
 }
 
 static CTypeDescrObject *new_void_type(void)
diff --git a/new/parse_c_type.c b/new/parse_c_type.c
--- a/new/parse_c_type.c
+++ b/new/parse_c_type.c
@@ -330,7 +330,7 @@
                 length = strtoull(tok->p, NULL, 10);
             else
                 length = strtoul(tok->p, NULL, 10);
-            if (errno == ERANGE)
+            if (errno == ERANGE || ((ssize_t)length) < 0)
                 return parse_error(tok, "number too large");
             next_token(tok);
 
diff --git a/new/realize_c_type.c b/new/realize_c_type.c
--- a/new/realize_c_type.c
+++ b/new/realize_c_type.c
@@ -34,8 +34,9 @@
                                  _cffi_opcode_t opcodes[], int index)
 {
     CTypeDescrObject *ct;
-    CTypeDescrObject *x, *y;
+    CTypeDescrObject *x, *y, *z;
     _cffi_opcode_t op = opcodes[index];
+    Py_ssize_t length = -1;
 
     if ((((uintptr_t)op) & 1) == 0) {
         ct = (CTypeDescrObject *)op;
@@ -60,6 +61,25 @@
         Py_DECREF(y);
         break;
 
+    case _CFFI_OP_ARRAY:
+        length = (Py_ssize_t)opcodes[_CFFI_GETARG(op) + 1];
+        /* fall-through */
+    case _CFFI_OP_OPEN_ARRAY:
+        y = realize_c_type(ctx, opcodes, _CFFI_GETARG(op));
+        if (y == NULL)
+            return NULL;
+        z = new_pointer_type(y);
+        Py_DECREF(y);
+        if (z == NULL)
+            return NULL;
+        x = new_array_type(z, length);
+        Py_DECREF(z);
+        break;
+
+    case _CFFI_OP_NOOP:
+        x = realize_c_type(ctx, opcodes, _CFFI_GETARG(op));
+        break;
+
     default:
         PyErr_Format(PyExc_NotImplementedError, "op=%d", (int)_CFFI_GETOP(op));
         return NULL;
diff --git a/new/test_realize_c_type.py b/new/test_realize_c_type.py
--- a/new/test_realize_c_type.py
+++ b/new/test_realize_c_type.py
@@ -14,3 +14,10 @@
 def test_int_star():
     check("int")
     check("int *")
+    check("int*", "int *")
+
+def test_noop():
+    check("int(*)", "int *")
+
+def test_array():
+    check("int[5]")

From noreply at buildbot.pypy.org  Tue Apr 14 17:28:47 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 14 Apr 2015 17:28:47 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Long
Message-ID: <20150414152847.5F1051C06CD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1703:2be5c31726b0
Date: 2015-04-14 16:16 +0200
http://bitbucket.org/cffi/cffi/changeset/2be5c31726b0/

Log:	Long

diff --git a/new/realize_c_type.c b/new/realize_c_type.c
--- a/new/realize_c_type.c
+++ b/new/realize_c_type.c
@@ -16,6 +16,10 @@
         x = new_primitive_type("int");
         break;
 
+    case _CFFI_PRIM_LONG:
+        x = new_primitive_type("long");
+        break;
+
     default:
         PyErr_Format(PyExc_NotImplementedError, "prim=%d", num);
         return NULL;
diff --git a/new/test_realize_c_type.py b/new/test_realize_c_type.py
--- a/new/test_realize_c_type.py
+++ b/new/test_realize_c_type.py
@@ -15,6 +15,8 @@
     check("int")
     check("int *")
     check("int*", "int *")
+    check("long int", "long")
+    check("long")
 
 def test_noop():
     check("int(*)", "int *")

From noreply at buildbot.pypy.org  Tue Apr 14 17:28:48 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 14 Apr 2015 17:28:48 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: in-progress: loading C extension
 modules and calling functions from them
Message-ID: <20150414152848.838631C06CD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1704:102edaf76b7c
Date: 2015-04-14 17:29 +0200
http://bitbucket.org/cffi/cffi/changeset/102edaf76b7c/

Log:	in-progress: loading C extension modules and calling functions from
	them

diff --git a/new/cffi1_module.c b/new/cffi1_module.c
--- a/new/cffi1_module.c
+++ b/new/cffi1_module.c
@@ -10,7 +10,7 @@
 static PyObject *FFIError;
 
 #include "ffi_obj.c"
-//#include "lib_obj.c"
+#include "lib_obj.c"
 
 
 static int init_ffi_lib(PyObject *m)
@@ -32,6 +32,9 @@
     Py_INCREF(&FFI_Type);
     if (PyModule_AddObject(m, "FFI", (PyObject *)&FFI_Type) < 0)
         return -1;
+    Py_INCREF(&Lib_Type);
+    if (PyModule_AddObject(m, "Lib", (PyObject *)&Lib_Type) < 0)
+        return -1;
 
     return 0;
 }
@@ -47,5 +50,9 @@
     if (ffi == NULL || PyModule_AddObject(m, "ffi", (PyObject *)ffi) < 0)
         return -1;
 
+    LibObject *lib = lib_internal_new(ctx, module_name);
+    if (lib == NULL || PyModule_AddObject(m, "lib", (PyObject *)lib) < 0)
+        return -1;
+
     return 0;
 }
diff --git a/new/lib_obj.c b/new/lib_obj.c
--- a/new/lib_obj.c
+++ b/new/lib_obj.c
@@ -12,69 +12,143 @@
    __getattr__ which returns C globals, functions and constants.  It
    raises AttributeError for anything else, like '__class__'.
 
-   A Lib object has internally a reference back to the FFI object,
-   which holds the _cffi_type_context_s used to create lazily the
-   objects returned by __getattr__.  For a dlopen()ed Lib object, all
-   the 'address' fields in _cffi_global_s are NULL, and instead
-   dlsym() is used lazily on the l_dl_lib.
+   A Lib object has got a reference to the _cffi_type_context_s
+   structure, which is used to create lazily the objects returned by
+   __getattr__.  For a dlopen()ed Lib object, all the 'address' fields
+   in _cffi_global_s are NULL, and instead dlsym() is used lazily on
+   the l_dl_lib.
 */
 
+struct CPyExtFunc_s {
+    PyMethodDef md;
+    const struct _cffi_type_context_s *ctx;
+    int type_index;
+};
+
 struct LibObject_s {
     PyObject_HEAD
+    const struct _cffi_type_context_s *l_ctx;  /* ctx object */
     PyObject *l_dict;           /* content, built lazily */
-    struct FFIObject_s *l_ffi;  /* ffi object */
     void *l_dl_lib;             /* the result of 'dlopen()', or NULL */
+    PyObject *l_libname;        /* some string that gives the name of the lib */
 };
 
-#define ZefLib_Check(ob)  ((Py_TYPE(ob) == &ZefLib_Type))
+#define LibObject_Check(ob)  ((Py_TYPE(ob) == &Lib_Type))
 
-static void lib_dealloc(ZefLibObject *lib)
+static int lib_close(LibObject *lib);    /* forward */
+
+static void lib_dealloc(LibObject *lib)
 {
     (void)lib_close(lib);
+    Py_DECREF(lib->l_dict);
+    Py_DECREF(lib->l_libname);
     PyObject_Del(lib);
 }
 
-static PyObject *lib_repr(ZefLibObject *lib)
+static PyObject *lib_repr(LibObject *lib)
 {
-    return PyText_FromFormat("",
-                             lib->l_libname,
-                             lib->l_dl_lib == NULL ? " (closed)" : "");
+    return PyText_FromFormat("",
+                             PyText_AS_UTF8(lib->l_libname));
 }
 
-static PyObject *lib_findattr(ZefLibObject *lib, PyObject *name, PyObject *exc)
+static PyObject *lib_build_cpython_func(LibObject *lib,
+                                        const struct _cffi_global_s *g,
+                                        const char *s, int flags)
+{
+    /* xxx the few bytes of memory we allocate here leak, but it's a
+       minor concern because it should only occur for CPYTHON_BLTN.
+       There is one per real C function in a CFFI C extension module.
+       CPython never unloads its C extension modules anyway.
+    */
+    struct CPyExtFunc_s *xfunc = calloc(1, sizeof(struct CPyExtFunc_s));
+    if (xfunc == NULL)
+        goto no_memory;
+
+    xfunc->md.ml_meth = (PyCFunction)g->address;
+    xfunc->md.ml_flags = flags;
+    xfunc->md.ml_name = strdup(s);
+    /*xfunc->md.ml_doc = ... */
+    if (xfunc->md.ml_name == NULL)
+        goto no_memory;
+
+    xfunc->ctx = lib->l_ctx;
+    xfunc->type_index = _CFFI_GETARG(g->type_op);
+
+    return PyCFunction_NewEx(&xfunc->md, NULL, lib->l_libname);
+
+ no_memory:
+    PyErr_NoMemory();
+    return NULL;
+}
+
+static PyObject *lib_build_and_cache_attr(LibObject *lib, PyObject *name)
 {
     /* does not return a new reference! */
 
-    if (lib->l_dict == NULL) {
-        PyErr_Format(ZefError, "lib '%.200s' was closed", lib->l_libname);
+    if (lib->l_ctx == NULL) {
+        PyErr_Format(FFIError, "lib '%.200s' is already closed",
+                     PyText_AS_UTF8(lib->l_libname));
         return NULL;
     }
 
-    PyObject *x = PyDict_GetItem(lib->l_dict, name);
-    if (x == NULL) {
-        PyErr_Format(exc,
+    char *s = PyText_AsUTF8(name);
+    if (s == NULL)
+        return NULL;
+
+    int index = search_in_globals(lib->l_ctx, s, strlen(s));
+    if (index < 0) {
+        PyErr_Format(PyExc_AttributeError,
                      "lib '%.200s' has no function,"
-                     " global variable or constant '%.200s'",
-                     lib->l_libname,
+                     " global variable or constant named '%.200s'",
+                     PyText_AS_UTF8(lib->l_libname),
                      PyText_Check(name) ? PyText_AS_UTF8(name) : "?");
         return NULL;
     }
+
+    const struct _cffi_global_s *g = &lib->l_ctx->globals[index];
+    PyObject *x;
+
+    switch (_CFFI_GETOP(g->type_op)) {
+
+    case _CFFI_OP_CPYTHON_BLTN_V:
+        x = lib_build_cpython_func(lib, g, s, METH_VARARGS);
+        break;
+
+    case _CFFI_OP_CPYTHON_BLTN_N:
+        x = lib_build_cpython_func(lib, g, s, METH_NOARGS);
+        break;
+
+    case _CFFI_OP_CPYTHON_BLTN_O:
+        x = lib_build_cpython_func(lib, g, s, METH_O);
+        break;
+
+    default:
+        PyErr_SetString(PyExc_NotImplementedError, "in lib_build_attr");
+        return NULL;
+    }
+
+    if (x != NULL) {
+        int err = PyDict_SetItem(lib->l_dict, name, x);
+        Py_DECREF(x);
+        if (err < 0)     /* else there is still one ref left in the dict */
+            return NULL;
+    }
     return x;
 }
 
-static PyObject *lib_getattr(ZefLibObject *lib, PyObject *name)
+static PyObject *lib_getattr(LibObject *lib, PyObject *name)
 {
-    PyObject *x = lib_findattr(lib, name, PyExc_AttributeError);
+    PyObject *x = PyDict_GetItem(lib->l_dict, name);
     if (x == NULL)
-        return NULL;
+        x = lib_build_and_cache_attr(lib, name);
 
-    if (ZefGlobSupport_Check(x)) {
-        return read_global_var((ZefGlobSupportObject *)x);
-    }
-    Py_INCREF(x);
+    //if (ZefGlobSupport_Check(x)) {
+    //    return read_global_var((ZefGlobSupportObject *)x);
+    //}
     return x;
 }
 
+#if 0
 static int lib_setattr(ZefLibObject *lib, PyObject *name, PyObject *val)
 {
     PyObject *x = lib_findattr(lib, name, PyExc_AttributeError);
@@ -106,11 +180,12 @@
     {"__dir__",   lib_dir,  METH_NOARGS},
     {NULL,        NULL}           /* sentinel */
 };
+#endif
 
-static PyTypeObject ZefLib_Type = {
+static PyTypeObject Lib_Type = {
     PyVarObject_HEAD_INIT(NULL, 0)
-    "zeffir.Lib",
-    sizeof(ZefLibObject),
+    "cffi.Lib",
+    sizeof(LibObject),
     0,
     (destructor)lib_dealloc,                    /* tp_dealloc */
     0,                                          /* tp_print */
@@ -125,7 +200,11 @@
     0,                                          /* tp_call */
     0,                                          /* tp_str */
     (getattrofunc)lib_getattr,                  /* tp_getattro */
+#if 0 // XXX
     (setattrofunc)lib_setattr,                  /* tp_setattro */
+#else
+    0,
+#endif
     0,                                          /* tp_as_buffer */
     Py_TPFLAGS_DEFAULT,                         /* tp_flags */
     0,                                          /* tp_doc */
@@ -135,63 +214,68 @@
     0,                                          /* tp_weaklistoffset */
     0,                                          /* tp_iter */
     0,                                          /* tp_iternext */
+#if 0 // XXX
     lib_methods,                                /* tp_methods */
+#else
+    0,
+#endif
     0,                                          /* tp_members */
     0,                                          /* tp_getset */
     0,                                          /* tp_base */
     0,                                          /* tp_dict */
     0,                                          /* tp_descr_get */
     0,                                          /* tp_descr_set */
-    offsetof(ZefLibObject, l_dict),             /* tp_dictoffset */
+    offsetof(LibObject, l_dict),                /* tp_dictoffset */
 };
 
-static void lib_dlerror(ZefLibObject *lib)
+
+static void lib_dlerror(LibObject *lib)
 {
     char *error = dlerror();
     if (error == NULL)
         error = "(no error reported)";
-    PyErr_Format(PyExc_OSError, "%s: %s", lib->l_libname, error);
+    PyErr_Format(PyExc_OSError, "%s: %s", PyText_AS_UTF8(lib->l_libname),
+                 error);
 }
 
-static ZefLibObject *lib_create(PyObject *path)
+static int lib_close(LibObject *lib)
 {
-    ZefLibObject *lib;
+    void *dll;
+    lib->l_ctx = NULL;
+    PyDict_Clear(lib->l_dict);
 
-    lib = PyObject_New(ZefLibObject, &ZefLib_Type);
-    if (lib == NULL)
-        return NULL;
-
-    lib->l_dl_lib = NULL;
-    lib->l_libname = PyString_AsString(path);
-    Py_INCREF(path);
-    lib->l_libname_obj = path;
-    lib->l_dict = PyDict_New();
-    if (lib->l_dict == NULL) {
-        Py_DECREF(lib);
-        return NULL;
-    }
-
-    lib->l_dl_lib = dlopen(lib->l_libname, RTLD_LAZY);
-    if (lib->l_dl_lib == NULL) {
-        lib_dlerror(lib);
-        Py_DECREF(lib);
-        return NULL;
-    }
-    return lib;
-}
-
-static int lib_close(ZefLibObject *lib)
-{
-    void *dl_lib;
-    Py_CLEAR(lib->l_dict);
-
-    dl_lib = lib->l_dl_lib;
-    if (dl_lib != NULL) {
+    dll = lib->l_dl_lib;
+    if (dll != NULL) {
         lib->l_dl_lib = NULL;
-        if (dlclose(dl_lib) != 0) {
+        if (dlclose(dll) != 0) {
             lib_dlerror(lib);
             return -1;
         }
     }
     return 0;
 }
+
+static LibObject *lib_internal_new(const struct _cffi_type_context_s *ctx,
+                                   char *module_name)
+{
+    LibObject *lib;
+    PyObject *libname, *dict;
+
+    libname = PyString_FromString(module_name);
+    dict = PyDict_New();
+    if (libname == NULL || dict == NULL) {
+        Py_XDECREF(dict);
+        Py_XDECREF(libname);
+        return NULL;
+    }
+
+    lib = PyObject_New(LibObject, &Lib_Type);
+    if (lib == NULL)
+        return NULL;
+
+    lib->l_ctx = ctx;
+    lib->l_dict = dict;
+    lib->l_dl_lib = NULL;
+    lib->l_libname = libname;
+    return lib;
+}
diff --git a/new/manual.c b/new/manual.c
--- a/new/manual.c
+++ b/new/manual.c
@@ -80,8 +80,8 @@
 }
 
 static const struct _cffi_global_s _cffi_globals[] = {
-    { "foo42", &_cffi_f_foo42, _CFFI_OP(_CFFI_OP_CPYTHON_BLTN, 0) },
-    { "foo64", &_cffi_f_foo64, _CFFI_OP(_CFFI_OP_CPYTHON_BLTN, 4) },
+    { "foo42", &_cffi_f_foo42, _CFFI_OP(_CFFI_OP_CPYTHON_BLTN_V, 0) },
+    { "foo64", &_cffi_f_foo64, _CFFI_OP(_CFFI_OP_CPYTHON_BLTN_O, 4) },
 };
 
 static const struct _cffi_type_context_s _cffi_type_context = {
@@ -92,7 +92,7 @@
     NULL,
     NULL,
     NULL,
-    1,  /* num_globals */
+    2,  /* num_globals */
     0,
     0,
     0,
diff --git a/new/parse_c_type.c b/new/parse_c_type.c
--- a/new/parse_c_type.c
+++ b/new/parse_c_type.c
@@ -349,43 +349,33 @@
     return _CFFI_GETARG(result);
 }
 
-static int search_struct_union(const struct _cffi_type_context_s *ctx,
-                               const char *search, size_t search_len)
-{
-    int left = 0, right = ctx->num_structs_unions;
 
-    while (left < right) {
-        int middle = (left + right) / 2;
-        const char *src = ctx->structs_unions[middle].name;
-        int diff = strncmp(src, search, search_len);
-        if (diff == 0 && src[search_len] == '\0')
-            return middle;
-        else if (diff >= 0)
-            right = middle;
-        else
-            left = middle + 1;
-    }
-    return -1;
-}
+#define MAKE_SEARCH_FUNC(FIELD)                                 \
+  int search_in_##FIELD(const struct _cffi_type_context_s *ctx, \
+                        const char *search, size_t search_len)  \
+  {                                                             \
+      int left = 0, right = ctx->num_##FIELD;                   \
+                                                                \
+      while (left < right) {                                    \
+          int middle = (left + right) / 2;                      \
+          const char *src = ctx->FIELD[middle].name;            \
+          int diff = strncmp(src, search, search_len);          \
+          if (diff == 0 && src[search_len] == '\0')             \
+              return middle;                                    \
+          else if (diff >= 0)                                   \
+              right = middle;                                   \
+          else                                                  \
+              left = middle + 1;                                \
+      }                                                         \
+      return -1;                                                \
+  }
 
-static int search_typename(const struct _cffi_type_context_s *ctx,
-                           const char *search, size_t search_len)
-{
-    int left = 0, right = ctx->num_typenames;
+MAKE_SEARCH_FUNC(globals)
+MAKE_SEARCH_FUNC(structs_unions)
+MAKE_SEARCH_FUNC(typenames)
 
-    while (left < right) {
-        int middle = (left + right) / 2;
-        const char *src = ctx->typenames[middle].name;
-        int diff = strncmp(src, search, search_len);
-        if (diff == 0 && src[search_len] == '\0')
-            return middle;
-        else if (diff >= 0)
-            right = middle;
-        else
-            left = middle + 1;
-    }
-    return -1;
-}
+#undef MAKE_SEARCH_FUNC
+
 
 static int parse_complete(token_t *tok)
 {
@@ -512,7 +502,7 @@
             break;
         case TOK_IDENTIFIER:
         {
-            int n = search_typename(tok->info->ctx, tok->p, tok->size);
+            int n = search_in_typenames(tok->info->ctx, tok->p, tok->size);
             if (n < 0)
                 return parse_error(tok, "undefined type name");
 
@@ -527,7 +517,7 @@
             if (tok->kind != TOK_IDENTIFIER)
                 return parse_error(tok, "struct or union name expected");
 
-            int n = search_struct_union(tok->info->ctx, tok->p, tok->size);
+            int n = search_in_structs_unions(tok->info->ctx, tok->p, tok->size);
             if (n < 0)
                 return parse_error(tok, "undefined struct/union name");
             if (((tok->info->ctx->structs_unions[n].flags & CT_UNION) != 0)
diff --git a/new/parse_c_type.h b/new/parse_c_type.h
--- a/new/parse_c_type.h
+++ b/new/parse_c_type.h
@@ -18,7 +18,9 @@
 #define _CFFI_OP_FUNCTION_END   17
 #define _CFFI_OP_NOOP           19
 #define _CFFI_OP_BITFIELD       21
-#define _CFFI_OP_CPYTHON_BLTN   23
+#define _CFFI_OP_CPYTHON_BLTN_V 23   // varargs
+#define _CFFI_OP_CPYTHON_BLTN_N 25   // noargs
+#define _CFFI_OP_CPYTHON_BLTN_O 27   // O  (i.e. a single arg)
 
 #define _CFFI_PRIM_VOID          0
 #define _CFFI_PRIM_BOOL          1
@@ -103,3 +105,5 @@
 };
 
 int parse_c_type(struct _cffi_parse_info_s *info, const char *input);
+int search_in_globals(const struct _cffi_type_context_s *ctx,
+                      const char *search, size_t search_len);

From noreply at buildbot.pypy.org  Tue Apr 14 18:18:31 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 14 Apr 2015 18:18:31 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: in-progress: decoding function types
 while only having a CT_FUNCTIONPTR...
Message-ID: <20150414161831.CD18A1C0661@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1705:e725f4ea7673
Date: 2015-04-14 18:19 +0200
http://bitbucket.org/cffi/cffi/changeset/e725f4ea7673/

Log:	in-progress: decoding function types while only having a
	CT_FUNCTIONPTR...

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -1902,7 +1902,7 @@
     return cd->c_data + i * cd->c_type->ct_itemdescr->ct_size;
 }
 
-static CTypeDescrObject *
+static PyObject *
 new_array_type(CTypeDescrObject *ctptr, Py_ssize_t length);   /* forward */
 
 static CTypeDescrObject *
@@ -1968,7 +1968,7 @@
         return NULL;
 
     if (ct->ct_stuff == NULL) {
-        ct->ct_stuff = (PyObject *)new_array_type(ct, -1);
+        ct->ct_stuff = new_array_type(ct, -1);
         if (ct->ct_stuff == NULL)
             return NULL;
     }
@@ -2269,18 +2269,13 @@
 static cif_description_t *
 fb_prepare_cif(PyObject *fargs, CTypeDescrObject *, ffi_abi);      /*forward*/
 
-static PyObject *
-b_new_primitive_type(PyObject *self, PyObject *args);              /*forward*/
+static PyObject *new_primitive_type(const char *name);             /*forward*/
 
 static CTypeDescrObject *_get_ct_int(void)
 {
     static CTypeDescrObject *ct_int = NULL;
     if (ct_int == NULL) {
-        PyObject *args = Py_BuildValue("(s)", "int");
-        if (args == NULL)
-            return NULL;
-        ct_int = (CTypeDescrObject *)b_new_primitive_type(NULL, args);
-        Py_DECREF(args);
+        ct_int = (CTypeDescrObject *)new_primitive_type("int");
     }
     return ct_int;
 }
@@ -3338,7 +3333,7 @@
 
 /************************************************************/
 
-static CTypeDescrObject *new_primitive_type(const char *name)
+static PyObject *new_primitive_type(const char *name)
 {
 #define ENUM_PRIMITIVE_TYPES                                    \
        EPTYPE(c, char, CT_PRIMITIVE_CHAR)                       \
@@ -3483,7 +3478,7 @@
             td->ct_flags |= CT_PRIMITIVE_FITS_LONG;
     }
     td->ct_name_position = strlen(td->ct_name);
-    return td;
+    return (PyObject *)td;
 
  bad_ffi_type:
     PyErr_Format(PyExc_NotImplementedError,
@@ -3498,10 +3493,10 @@
     char *name;
     if (!PyArg_ParseTuple(args, "s:new_primitive_type", &name))
         return NULL;
-    return (PyObject *)new_primitive_type(name);
-}
-
-static CTypeDescrObject *new_pointer_type(CTypeDescrObject *ctitem)
+    return new_primitive_type(name);
+}
+
+static PyObject *new_pointer_type(CTypeDescrObject *ctitem)
 {
     CTypeDescrObject *td;
     const char *extra;
@@ -3525,7 +3520,7 @@
         ((ctitem->ct_flags & CT_PRIMITIVE_CHAR) &&
          ctitem->ct_size == sizeof(char)))
         td->ct_flags |= CT_CAST_ANYTHING;   /* 'void *' or 'char *' only */
-    return td;
+    return (PyObject *)td;
 }
 
 static PyObject *b_new_pointer_type(PyObject *self, PyObject *args)
@@ -3534,7 +3529,7 @@
     if (!PyArg_ParseTuple(args, "O!:new_pointer_type",
                           &CTypeDescr_Type, &ctitem))
         return NULL;
-    return (PyObject *)new_pointer_type(ctitem);
+    return new_pointer_type(ctitem);
 }
 
 static PyObject *b_new_array_type(PyObject *self, PyObject *args)
@@ -3558,10 +3553,10 @@
             return NULL;
         }
     }
-    return (PyObject *)new_array_type(ctptr, length);
-}
-
-static CTypeDescrObject *
+    return new_array_type(ctptr, length);
+}
+
+static PyObject *
 new_array_type(CTypeDescrObject *ctptr, Py_ssize_t length)
 {
     CTypeDescrObject *td, *ctitem;
@@ -3606,10 +3601,10 @@
     td->ct_size = arraysize;
     td->ct_length = length;
     td->ct_flags = flags;
-    return td;
-}
-
-static CTypeDescrObject *new_void_type(void)
+    return (PyObject *)td;
+}
+
+static PyObject *new_void_type(void)
 {
     int name_size = strlen("void") + 1;
     CTypeDescrObject *td = ctypedescr_new(name_size);
@@ -3620,12 +3615,12 @@
     td->ct_size = -1;
     td->ct_flags = CT_VOID | CT_IS_OPAQUE;
     td->ct_name_position = strlen("void");
-    return td;
+    return (PyObject *)td;
 }
 
 static PyObject *b_new_void_type(PyObject *self, PyObject *args)
 {
-    return (PyObject *)new_void_type();
+    return new_void_type();
 }
 
 static PyObject *_b_struct_or_union_type(const char *name, int flag)
@@ -4389,22 +4384,15 @@
     return NULL;
 }
 
-static PyObject *b_new_function_type(PyObject *self, PyObject *args)
-{
-    PyObject *fargs, *fabiobj;
-    CTypeDescrObject *fresult;
+static PyObject *new_function_type(PyObject *fargs,   /* tuple */
+                                   CTypeDescrObject *fresult,
+                                   int ellipsis, int fabi)
+{
+    PyObject *fabiobj;
     CTypeDescrObject *fct;
-    int ellipsis = 0, fabi = FFI_DEFAULT_ABI;
     struct funcbuilder_s funcbuilder;
     Py_ssize_t i;
 
-    if (!PyArg_ParseTuple(args, "O!O!|ii:new_function_type",
-                          &PyTuple_Type, &fargs,
-                          &CTypeDescr_Type, &fresult,
-                          &ellipsis,
-                          &fabi))
-        return NULL;
-
     if ((fresult->ct_size < 0 && !(fresult->ct_flags & CT_VOID)) ||
         (fresult->ct_flags & CT_ARRAY)) {
         char *msg;
@@ -4468,6 +4456,22 @@
     return NULL;
 }
 
+static PyObject *b_new_function_type(PyObject *self, PyObject *args)
+{
+    PyObject *fargs;
+    CTypeDescrObject *fresult;
+    int ellipsis = 0, fabi = FFI_DEFAULT_ABI;
+
+    if (!PyArg_ParseTuple(args, "O!O!|ii:new_function_type",
+                          &PyTuple_Type, &fargs,
+                          &CTypeDescr_Type, &fresult,
+                          &ellipsis,
+                          &fabi))
+        return NULL;
+
+    return new_function_type(fargs, fresult, ellipsis, fabi);
+}
+
 static int convert_from_object_fficallback(char *result,
                                            CTypeDescrObject *ctype,
                                            PyObject *pyobj)
diff --git a/new/ffi_obj.c b/new/ffi_obj.c
--- a/new/ffi_obj.c
+++ b/new/ffi_obj.c
@@ -101,19 +101,23 @@
                          input_text, spaces);
             return NULL;
         }
-        CTypeDescrObject *ct = realize_c_type(ffi->info.ctx,
-                                              ffi->info.output, index);
+        PyObject *ct = realize_c_type(ffi->info.ctx,
+                                      ffi->info.output, index);
         if (ct == NULL)
             return NULL;
-        x = PyDict_GetItemString(ffi->types_dict, ct->ct_name);
-        if (x != NULL) {
+
+        char *normalized_text = ((CTypeDescrObject *)ct)->ct_name;
+        x = PyDict_GetItemString(ffi->types_dict, normalized_text);
+        if (x == NULL) {
+            PyDict_SetItemString(ffi->types_dict, normalized_text, ct);
+        }
+        else {
             Py_INCREF(x);
             Py_DECREF(ct);
-            ct = (CTypeDescrObject *)x;
+            ct = x;
         }
-        PyDict_SetItemString(ffi->types_dict, ct->ct_name, (PyObject *)ct);
-        PyDict_SetItem(ffi->types_dict, arg, (PyObject *)ct);
-        return ct;
+        PyDict_SetItem(ffi->types_dict, arg, ct);
+        return (CTypeDescrObject *)ct;
     }
     else if ((accept & ACCEPT_CTYPE) && CTypeDescr_Check(arg)) {
         return (CTypeDescrObject *)arg;
diff --git a/new/lib_obj.c b/new/lib_obj.c
--- a/new/lib_obj.c
+++ b/new/lib_obj.c
@@ -66,7 +66,7 @@
 
     xfunc->md.ml_meth = (PyCFunction)g->address;
     xfunc->md.ml_flags = flags;
-    xfunc->md.ml_name = strdup(s);
+    xfunc->md.ml_name = g->name;
     /*xfunc->md.ml_doc = ... */
     if (xfunc->md.ml_name == NULL)
         goto no_memory;
diff --git a/new/realize_c_type.c b/new/realize_c_type.c
--- a/new/realize_c_type.c
+++ b/new/realize_c_type.c
@@ -1,10 +1,10 @@
 
-static CTypeDescrObject *all_primitives[_CFFI__NUM_PRIM];
+static PyObject *all_primitives[_CFFI__NUM_PRIM];
 
 
-CTypeDescrObject *build_primitive_type(int num)
+PyObject *build_primitive_type(int num)
 {
-    CTypeDescrObject *x;
+    PyObject *x;
 
     switch (num) {
 
@@ -30,22 +30,39 @@
 }
 
 
+static PyObject *
+_realize_c_type(const struct _cffi_type_context_s *ctx,
+                _cffi_opcode_t opcodes[], int index);  /* forward */
+
+
 /* Interpret an opcodes[] array.  If opcodes == ctx->types, store all
    the intermediate types back in the opcodes[].  Returns a new
    reference.
 */
-CTypeDescrObject *realize_c_type(const struct _cffi_type_context_s *ctx,
-                                 _cffi_opcode_t opcodes[], int index)
+static PyObject *
+realize_c_type(const struct _cffi_type_context_s *ctx,
+               _cffi_opcode_t opcodes[], int index)
 {
-    CTypeDescrObject *ct;
-    CTypeDescrObject *x, *y, *z;
+    PyObject *x = _realize_c_type(ctx, opcodes, index);
+    if (x != NULL && !CTypeDescr_Check(x)) {
+        PyErr_SetString(PyExc_NotImplementedError, "unexpected type");
+        x = NULL;
+    }
+    return x;
+}
+
+static PyObject *
+_realize_c_type(const struct _cffi_type_context_s *ctx,
+                _cffi_opcode_t opcodes[], int index)
+{
+    PyObject *x, *y, *z;
     _cffi_opcode_t op = opcodes[index];
     Py_ssize_t length = -1;
 
     if ((((uintptr_t)op) & 1) == 0) {
-        ct = (CTypeDescrObject *)op;
-        Py_INCREF(ct);
-        return ct;
+        x = (PyObject *)op;
+        Py_INCREF(x);
+        return x;
     }
 
     switch (_CFFI_GETOP(op)) {
@@ -58,10 +75,17 @@
         break;
 
     case _CFFI_OP_POINTER:
-        y = realize_c_type(ctx, opcodes, _CFFI_GETARG(op));
+        y = _realize_c_type(ctx, opcodes, _CFFI_GETARG(op));
         if (y == NULL)
             return NULL;
-        x = new_pointer_type(y);
+        if (CTypeDescr_Check(y)) {
+            x = new_pointer_type((CTypeDescrObject *)y);
+        }
+        else {
+            assert(PyTuple_Check(y));   /* from _CFFI_OP_FUNCTION */
+            x = PyTuple_GET_ITEM(y, 0);
+            Py_INCREF(x);
+        }
         Py_DECREF(y);
         break;
 
@@ -72,16 +96,36 @@
         y = realize_c_type(ctx, opcodes, _CFFI_GETARG(op));
         if (y == NULL)
             return NULL;
-        z = new_pointer_type(y);
+        z = new_pointer_type((CTypeDescrObject *)y);
         Py_DECREF(y);
         if (z == NULL)
             return NULL;
-        x = new_array_type(z, length);
+        x = new_array_type((CTypeDescrObject *)z, length);
         Py_DECREF(z);
         break;
 
+    case _CFFI_OP_FUNCTION:
+    {
+        PyObject *fargs;
+
+        y = realize_c_type(ctx, opcodes, _CFFI_GETARG(op));
+        if (y == NULL)
+            return NULL;
+        fargs = PyTuple_New(0);  // XXX NULL
+        z = new_function_type(fargs, (CTypeDescrObject *)y, 0, FFI_DEFAULT_ABI);
+        Py_DECREF(fargs);
+        Py_DECREF(y);
+        if (z == NULL)
+            return NULL;
+
+        x = PyTuple_Pack(1, z);   /* hack: hide the CT_FUNCTIONPTR.  it will
+                                     be revealed again by the OP_POINTER */
+        Py_DECREF(z);
+        break;
+    }
+
     case _CFFI_OP_NOOP:
-        x = realize_c_type(ctx, opcodes, _CFFI_GETARG(op));
+        x = _realize_c_type(ctx, opcodes, _CFFI_GETARG(op));
         break;
 
     default:
diff --git a/new/test_realize_c_type.py b/new/test_realize_c_type.py
--- a/new/test_realize_c_type.py
+++ b/new/test_realize_c_type.py
@@ -23,3 +23,6 @@
 
 def test_array():
     check("int[5]")
+
+def test_funcptr():
+    check("int(*)(long)")

From noreply at buildbot.pypy.org  Tue Apr 14 18:47:59 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 14 Apr 2015 18:47:59 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Process argument types
Message-ID: <20150414164759.64B421C06CA@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1706:4070e62efc66
Date: 2015-04-14 18:48 +0200
http://bitbucket.org/cffi/cffi/changeset/4070e62efc66/

Log:	Process argument types

diff --git a/new/realize_c_type.c b/new/realize_c_type.c
--- a/new/realize_c_type.c
+++ b/new/realize_c_type.c
@@ -107,11 +107,34 @@
     case _CFFI_OP_FUNCTION:
     {
         PyObject *fargs;
+        int i, base_index, num_args;
 
         y = realize_c_type(ctx, opcodes, _CFFI_GETARG(op));
         if (y == NULL)
             return NULL;
-        fargs = PyTuple_New(0);  // XXX NULL
+
+        base_index = index + 1;
+        num_args = 0;
+        while (_CFFI_GETOP(opcodes[base_index + num_args]) !=
+                   _CFFI_OP_FUNCTION_END)
+            num_args++;
+
+        fargs = PyTuple_New(num_args);
+        if (fargs == NULL) {
+            Py_DECREF(y);
+            return NULL;
+        }
+
+        for (i = 0; i < num_args; i++) {
+            z = realize_c_type(ctx, opcodes, base_index + i);
+            if (z == NULL) {
+                Py_DECREF(fargs);
+                Py_DECREF(y);
+                return NULL;
+            }
+            PyTuple_SET_ITEM(fargs, i, z);
+        }
+
         z = new_function_type(fargs, (CTypeDescrObject *)y, 0, FFI_DEFAULT_ABI);
         Py_DECREF(fargs);
         Py_DECREF(y);

From noreply at buildbot.pypy.org  Tue Apr 14 18:54:33 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 14 Apr 2015 18:54:33 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: clarify error message
Message-ID: <20150414165433.F41CB1C06CA@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1707:3ce0431464af
Date: 2015-04-14 18:55 +0200
http://bitbucket.org/cffi/cffi/changeset/3ce0431464af/

Log:	clarify error message

diff --git a/new/cffi1_module.c b/new/cffi1_module.c
--- a/new/cffi1_module.c
+++ b/new/cffi1_module.c
@@ -1,3 +1,5 @@
+static PyObject *FFIError;
+
 #include "parse_c_type.c"
 #include "realize_c_type.c"
 
@@ -7,8 +9,6 @@
 static PyTypeObject FFI_Type;   /* forward */
 static PyTypeObject Lib_Type;   /* forward */
 
-static PyObject *FFIError;
-
 #include "ffi_obj.c"
 #include "lib_obj.c"
 
diff --git a/new/realize_c_type.c b/new/realize_c_type.c
--- a/new/realize_c_type.c
+++ b/new/realize_c_type.c
@@ -31,8 +31,8 @@
 
 
 static PyObject *
-_realize_c_type(const struct _cffi_type_context_s *ctx,
-                _cffi_opcode_t opcodes[], int index);  /* forward */
+_realize_c_type_or_func(const struct _cffi_type_context_s *ctx,
+                        _cffi_opcode_t opcodes[], int index);  /* forward */
 
 
 /* Interpret an opcodes[] array.  If opcodes == ctx->types, store all
@@ -43,17 +43,25 @@
 realize_c_type(const struct _cffi_type_context_s *ctx,
                _cffi_opcode_t opcodes[], int index)
 {
-    PyObject *x = _realize_c_type(ctx, opcodes, index);
-    if (x != NULL && !CTypeDescr_Check(x)) {
-        PyErr_SetString(PyExc_NotImplementedError, "unexpected type");
-        x = NULL;
+    PyObject *x = _realize_c_type_or_func(ctx, opcodes, index);
+    if (x == NULL || CTypeDescr_Check(x)) {
+        return x;
     }
-    return x;
+    else {
+        PyObject *y;
+        assert(PyTuple_Check(x));
+        y = PyTuple_GET_ITEM(x, 0);
+        PyErr_Format(FFIError, "the type '%s' is a function type, not a "
+                               "pointer-to-function type",
+                     ((CTypeDescrObject *)y)->ct_name);
+        Py_DECREF(x);
+        return NULL;
+    }
 }
 
 static PyObject *
-_realize_c_type(const struct _cffi_type_context_s *ctx,
-                _cffi_opcode_t opcodes[], int index)
+_realize_c_type_or_func(const struct _cffi_type_context_s *ctx,
+                        _cffi_opcode_t opcodes[], int index)
 {
     PyObject *x, *y, *z;
     _cffi_opcode_t op = opcodes[index];
@@ -75,7 +83,7 @@
         break;
 
     case _CFFI_OP_POINTER:
-        y = _realize_c_type(ctx, opcodes, _CFFI_GETARG(op));
+        y = _realize_c_type_or_func(ctx, opcodes, _CFFI_GETARG(op));
         if (y == NULL)
             return NULL;
         if (CTypeDescr_Check(y)) {
@@ -148,7 +156,7 @@
     }
 
     case _CFFI_OP_NOOP:
-        x = _realize_c_type(ctx, opcodes, _CFFI_GETARG(op));
+        x = _realize_c_type_or_func(ctx, opcodes, _CFFI_GETARG(op));
         break;
 
     default:

From noreply at buildbot.pypy.org  Tue Apr 14 19:05:45 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 14 Apr 2015 19:05:45 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Test and fix for the error message
Message-ID: <20150414170545.A28B21C1DCF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1708:30a839e74f3d
Date: 2015-04-14 19:06 +0200
http://bitbucket.org/cffi/cffi/changeset/30a839e74f3d/

Log:	Test and fix for the error message

diff --git a/new/realize_c_type.c b/new/realize_c_type.c
--- a/new/realize_c_type.c
+++ b/new/realize_c_type.c
@@ -51,9 +51,13 @@
         PyObject *y;
         assert(PyTuple_Check(x));
         y = PyTuple_GET_ITEM(x, 0);
-        PyErr_Format(FFIError, "the type '%s' is a function type, not a "
-                               "pointer-to-function type",
-                     ((CTypeDescrObject *)y)->ct_name);
+        char *text1 = ((CTypeDescrObject *)y)->ct_name;
+        char *text2 = text1 + ((CTypeDescrObject *)y)->ct_name_position + 1;
+        assert(text2[-3] == '(');
+        text2[-3] = '\0';
+        PyErr_Format(FFIError, "the type '%s%s' is a function type, not a "
+                               "pointer-to-function type", text1, text2);
+        text2[-3] = '(';
         Py_DECREF(x);
         return NULL;
     }
diff --git a/new/test_realize_c_type.py b/new/test_realize_c_type.py
--- a/new/test_realize_c_type.py
+++ b/new/test_realize_c_type.py
@@ -1,11 +1,17 @@
+import py
 
 
-def check(input, expected_output=None):
+def check(input, expected_output=None, expected_ffi_error=False):
     import _cffi1_backend
     ffi = _cffi1_backend.FFI()
-    ct = ffi.typeof(input)
-    assert isinstance(ct, ffi.CType)
-    assert ct.cname == (expected_output or input)
+    if not expected_ffi_error:
+        ct = ffi.typeof(input)
+        assert isinstance(ct, ffi.CType)
+        assert ct.cname == (expected_output or input)
+    else:
+        e = py.test.raises(ffi.error, ffi.typeof, input)
+        if isinstance(expected_ffi_error, str):
+            assert str(e.value) == expected_ffi_error
 
 def test_void():
     check("void", "void")
@@ -26,3 +32,7 @@
 
 def test_funcptr():
     check("int(*)(long)")
+    check("int(long)", expected_ffi_error="the type 'int(long)' is a"
+          " function type, not a pointer-to-function type")
+    check("int(void)", expected_ffi_error="the type 'int()' is a"
+          " function type, not a pointer-to-function type")

From noreply at buildbot.pypy.org  Tue Apr 14 19:07:08 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 14 Apr 2015 19:07:08 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: passing test
Message-ID: <20150414170708.E11BD1C1DFF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1709:513fcc92ba99
Date: 2015-04-14 19:07 +0200
http://bitbucket.org/cffi/cffi/changeset/513fcc92ba99/

Log:	passing test

diff --git a/new/test_realize_c_type.py b/new/test_realize_c_type.py
--- a/new/test_realize_c_type.py
+++ b/new/test_realize_c_type.py
@@ -36,3 +36,8 @@
           " function type, not a pointer-to-function type")
     check("int(void)", expected_ffi_error="the type 'int()' is a"
           " function type, not a pointer-to-function type")
+
+def test_funcptr_rewrite_args():
+    check("int(*)(int(int))", "int(*)(int(*)(int))")
+    check("int(*)(long[])", "int(*)(long *)")
+    check("int(*)(long[5])", "int(*)(long *)")

From noreply at buildbot.pypy.org  Wed Apr 15 02:39:41 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Wed, 15 Apr 2015 02:39:41 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix whatsnew
Message-ID: <20150415003941.7BA3D1C0362@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: 
Changeset: r76790:e10aecf11072
Date: 2015-04-15 01:39 +0100
http://bitbucket.org/pypy/pypy/changeset/e10aecf11072/

Log:	fix whatsnew

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -21,3 +21,5 @@
 Make `*`-unpacking in RPython function calls completely equivalent to passing
 the tuple's elements as arguments. In other words, `f(*(a, b))` now behaves 
 exactly like `f(a, b)`.
+
+.. branch: issue2018

From noreply at buildbot.pypy.org  Wed Apr 15 02:41:31 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Wed, 15 Apr 2015 02:41:31 +0200 (CEST)
Subject: [pypy-commit] pypy callfamily: Move compute_at_fixpoint() to
	simplify()
Message-ID: <20150415004131.C89ED1C0369@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: callfamily
Changeset: r76791:50e806524e74
Date: 2015-03-31 19:54 +0100
http://bitbucket.org/pypy/pypy/changeset/50e806524e74/

Log:	Move compute_at_fixpoint() to simplify()

diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -213,8 +213,6 @@
             v = graph.getreturnvar()
             if v.annotation is None:
                 self.setbinding(v, annmodel.s_ImpossibleValue)
-        # policy-dependent computation
-        self.bookkeeper.compute_at_fixpoint()
 
     def validate(self):
         """Check that the annotation results are valid"""
@@ -309,6 +307,7 @@
                     graphs[graph] = True
         for graph in graphs:
             simplify.eliminate_empty_blocks(graph)
+        self.bookkeeper.compute_at_fixpoint()
         if block_subset is None:
             perform_normalizations(self)
 
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -1039,102 +1039,6 @@
         assert acc1 is acc2
         assert acc1.attrs.keys() == ['v1']
 
-    def test_simple_pbc_call(self):
-        def f1(x,y=0):
-            pass
-        def f2(x):
-            pass
-        def f3(x):
-            pass
-        def g(f):
-            f(1)
-        def h():
-            f1(1)
-            f1(1,2)
-            g(f2)
-            g(f3)
-
-        a = self.RPythonAnnotator()
-        s = a.build_types(h, [])
-
-        fdesc1 = a.bookkeeper.getdesc(f1)
-        fdesc2 = a.bookkeeper.getdesc(f2)
-        fdesc3 = a.bookkeeper.getdesc(f3)
-
-        fam1 = fdesc1.getcallfamily()
-        fam2 = fdesc2.getcallfamily()
-        fam3 = fdesc3.getcallfamily()
-
-        assert fam1 is not fam2
-        assert fam1 is not fam3
-        assert fam3 is fam2
-
-        gf1 = graphof(a, f1)
-        gf2 = graphof(a, f2)
-        gf3 = graphof(a, f3)
-
-        assert fam1.calltables == {(2, (), False): [{fdesc1: gf1}],
-                                   (1, (), False): [{fdesc1: gf1}]}
-        assert fam2.calltables == {(1, (), False): [{fdesc2: gf2, fdesc3: gf3}]}
-
-    def test_pbc_call_ins(self):
-        class A(object):
-            def m(self):
-                pass
-        class B(A):
-            def n(self):
-                pass
-        class C(A):
-            def __init__(self):
-                pass
-            def m(self):
-                pass
-        def f(x):
-            b = B()
-            c = C()
-            b.n()
-            if x:
-                a = b
-            else:
-                a = c
-            a.m()
-
-        a = self.RPythonAnnotator()
-        s = a.build_types(f, [bool])
-
-        clsdef = a.bookkeeper.getuniqueclassdef
-        bookkeeper = a.bookkeeper
-
-        def getmdesc(bmeth):
-            return bookkeeper.immutablevalue(bmeth).any_description()
-
-        mdescA_m = getmdesc(A().m)
-        mdescC_m = getmdesc(C().m)
-        mdescB_n = getmdesc(B().n)
-
-        assert mdescA_m.name == 'm' == mdescC_m.name
-        assert mdescB_n.name == 'n'
-
-        famA_m = mdescA_m.getcallfamily()
-        famC_m = mdescC_m.getcallfamily()
-        famB_n = mdescB_n.getcallfamily()
-
-        assert famA_m is famC_m
-        assert famB_n is not famA_m
-
-        gfB_n = graphof(a, B.n.im_func)
-        gfA_m = graphof(a, A.m.im_func)
-        gfC_m = graphof(a, C.m.im_func)
-
-        assert famB_n.calltables == {(1, (), False): [{mdescB_n.funcdesc: gfB_n}] }
-        assert famA_m.calltables == {(1, (), False): [{mdescA_m.funcdesc: gfA_m, mdescC_m.funcdesc: gfC_m }] }
-
-        mdescCinit = getmdesc(C().__init__)
-        famCinit = mdescCinit.getcallfamily()
-        gfCinit = graphof(a, C.__init__.im_func)
-
-        assert famCinit.calltables == {(1, (), False): [{mdescCinit.funcdesc: gfCinit}] }
-
     def test_isinstance_usigned(self):
         def f(x):
             return isinstance(x, r_uint)
@@ -2056,6 +1960,7 @@
         s_f = a.bookkeeper.immutablevalue(f)
         a.bookkeeper.emulate_pbc_call('f', s_f, [annmodel.SomeInteger(), annmodel.SomeInteger()])
         a.complete()
+        a.simplify()
 
         assert a.binding(graphof(a, f).getreturnvar()).knowntype == int
         fdesc = a.bookkeeper.getdesc(f)
@@ -3816,28 +3721,6 @@
         e = py.test.raises(Exception, a.build_types, f, [int])
         assert "field '_my_lst' was migrated" in str(e.value)
 
-    def test_call_classes_with_noarg_init(self):
-        class A:
-            foo = 21
-        class B(A):
-            foo = 22
-        class C(A):
-            def __init__(self):
-                self.foo = 42
-        class D(A):
-            def __init__(self):
-                self.foo = 43
-        def f(i):
-            if i == 1:
-                cls = B
-            elif i == 2:
-                cls = D
-            else:
-                cls = C
-            return cls().foo
-        a = self.RPythonAnnotator()
-        py.test.raises(Exception, a.build_types, f, [int])
-
     def test_range_variable_step(self):
         def g(n):
             return range(0, 10, n)
diff --git a/rpython/annotator/test/test_annsimplifyrpython.py b/rpython/annotator/test/test_annsimplifyrpython.py
--- a/rpython/annotator/test/test_annsimplifyrpython.py
+++ b/rpython/annotator/test/test_annsimplifyrpython.py
@@ -12,3 +12,122 @@
             parent.RPythonAnnotator.complete(self)
             if self.translator is not None:
                 self.simplify()
+
+    def test_simple_pbc_call(self):
+        def f1(x,y=0):
+            pass
+        def f2(x):
+            pass
+        def f3(x):
+            pass
+        def g(f):
+            f(1)
+        def h():
+            f1(1)
+            f1(1,2)
+            g(f2)
+            g(f3)
+
+        a = self.RPythonAnnotator()
+        s = a.build_types(h, [])
+
+        fdesc1 = a.bookkeeper.getdesc(f1)
+        fdesc2 = a.bookkeeper.getdesc(f2)
+        fdesc3 = a.bookkeeper.getdesc(f3)
+
+        fam1 = fdesc1.getcallfamily()
+        fam2 = fdesc2.getcallfamily()
+        fam3 = fdesc3.getcallfamily()
+
+        assert fam1 is not fam2
+        assert fam1 is not fam3
+        assert fam3 is fam2
+
+        gf1 = graphof(a, f1)
+        gf2 = graphof(a, f2)
+        gf3 = graphof(a, f3)
+
+        assert fam1.calltables == {(2, (), False): [{fdesc1: gf1}],
+                                   (1, (), False): [{fdesc1: gf1}]}
+        assert fam2.calltables == {(1, (), False): [{fdesc2: gf2, fdesc3: gf3}]}
+
+    def test_pbc_call_ins(self):
+        class A(object):
+            def m(self):
+                pass
+        class B(A):
+            def n(self):
+                pass
+        class C(A):
+            def __init__(self):
+                pass
+            def m(self):
+                pass
+        def f(x):
+            b = B()
+            c = C()
+            b.n()
+            if x:
+                a = b
+            else:
+                a = c
+            a.m()
+
+        a = self.RPythonAnnotator()
+        s = a.build_types(f, [bool])
+
+        clsdef = a.bookkeeper.getuniqueclassdef
+        bookkeeper = a.bookkeeper
+
+        def getmdesc(bmeth):
+            return bookkeeper.immutablevalue(bmeth).any_description()
+
+        mdescA_m = getmdesc(A().m)
+        mdescC_m = getmdesc(C().m)
+        mdescB_n = getmdesc(B().n)
+
+        assert mdescA_m.name == 'm' == mdescC_m.name
+        assert mdescB_n.name == 'n'
+
+        famA_m = mdescA_m.getcallfamily()
+        famC_m = mdescC_m.getcallfamily()
+        famB_n = mdescB_n.getcallfamily()
+
+        assert famA_m is famC_m
+        assert famB_n is not famA_m
+
+        gfB_n = graphof(a, B.n.im_func)
+        gfA_m = graphof(a, A.m.im_func)
+        gfC_m = graphof(a, C.m.im_func)
+
+        assert famB_n.calltables == {(1, (), False): [{mdescB_n.funcdesc: gfB_n}] }
+        assert famA_m.calltables == {(1, (), False): [{mdescA_m.funcdesc: gfA_m, mdescC_m.funcdesc: gfC_m }] }
+
+        mdescCinit = getmdesc(C().__init__)
+        famCinit = mdescCinit.getcallfamily()
+        gfCinit = graphof(a, C.__init__.im_func)
+
+        assert famCinit.calltables == {(1, (), False): [{mdescCinit.funcdesc: gfCinit}] }
+
+    def test_call_classes_with_noarg_init(self):
+        class A:
+            foo = 21
+        class B(A):
+            foo = 22
+        class C(A):
+            def __init__(self):
+                self.foo = 42
+        class D(A):
+            def __init__(self):
+                self.foo = 43
+        def f(i):
+            if i == 1:
+                cls = B
+            elif i == 2:
+                cls = D
+            else:
+                cls = C
+            return cls().foo
+        a = self.RPythonAnnotator()
+        with py.test.raises(Exception):
+            a.build_types(f, [int])

From noreply at buildbot.pypy.org  Wed Apr 15 02:41:33 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Wed, 15 Apr 2015 02:41:33 +0200 (CEST)
Subject: [pypy-commit] pypy callfamily: Don't bother analysing call families
 until they are needed
Message-ID: <20150415004133.2D8711C0369@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: callfamily
Changeset: r76792:00452e09acc2
Date: 2015-03-29 15:08 +0100
http://bitbucket.org/pypy/pypy/changeset/00452e09acc2/

Log:	Don't bother analysing call families until they are needed

diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -495,9 +495,6 @@
         annotations).
         """
         descs = list(pbc.descriptions)
-        first = descs[0]
-        first.mergecallfamilies(*descs[1:])
-
         if emulated is None:
             whence = self.position_key
             # fish the existing annotation for the result variable,
@@ -519,7 +516,7 @@
             return self.annotator.recursivecall(graph, whence, inputcells)
 
         results = []
-        for desc in descs:
+        for desc in pbc.descriptions:
             results.append(desc.pycall(schedule, args, s_previous_result, op))
         s_result = unionof(*results)
         return s_result
diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py
--- a/rpython/annotator/description.py
+++ b/rpython/annotator/description.py
@@ -351,6 +351,7 @@
         row = FunctionDesc.row_to_consider(descs, args, op)
         family = descs[0].getcallfamily()
         family.calltable_add_row(shape, row)
+        descs[0].mergecallfamilies(*descs[1:])
 
     @staticmethod
     def variant_for_call_site(bookkeeper, family, descs, args, op):
@@ -777,6 +778,8 @@
 
     @staticmethod
     def consider_call_site(descs, args, s_result, op):
+        descs[0].getcallfamily()
+        descs[0].mergecallfamilies(*descs[1:])
         from rpython.annotator.model import SomeInstance, SomePBC, s_None
         if len(descs) == 1:
             # call to a single class, look at the result annotation
@@ -916,6 +919,7 @@
         row = FunctionDesc.row_to_consider(descs, args, op)
         family = descs[0].getcallfamily()
         family.calltable_add_row(shape, row)
+        descs[0].mergecallfamilies(*descs[1:])
 
     def rowkey(self):
         # we are computing call families and call tables that always contain
@@ -1077,6 +1081,7 @@
         row = FunctionDesc.row_to_consider(descs, args, op)
         family = descs[0].getcallfamily()
         family.calltable_add_row(shape, row)
+        descs[0].mergecallfamilies(*descs[1:])
 
     def rowkey(self):
         return self.funcdesc

From noreply at buildbot.pypy.org  Wed Apr 15 02:41:34 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Wed, 15 Apr 2015 02:41:34 +0200 (CEST)
Subject: [pypy-commit] pypy callfamily: fix tests
Message-ID: <20150415004134.70A3B1C0369@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: callfamily
Changeset: r76793:6bee8ed11bb3
Date: 2015-04-14 20:38 +0100
http://bitbucket.org/pypy/pypy/changeset/6bee8ed11bb3/

Log:	fix tests

diff --git a/rpython/annotator/test/test_annsimplifyrpython.py b/rpython/annotator/test/test_annsimplifyrpython.py
--- a/rpython/annotator/test/test_annsimplifyrpython.py
+++ b/rpython/annotator/test/test_annsimplifyrpython.py
@@ -1,5 +1,8 @@
-import rpython.annotator.test.test_annrpython
-parent = rpython.annotator.test.test_annrpython.TestAnnotateTestCase
+import py
+
+
+from rpython.annotator.test.test_annrpython import graphof
+from rpython.annotator.test.test_annrpython import TestAnnotateTestCase as parent
 
 
 class TestAnnotateAndSimplifyTestCase(parent):
@@ -76,7 +79,6 @@
         a = self.RPythonAnnotator()
         s = a.build_types(f, [bool])
 
-        clsdef = a.bookkeeper.getuniqueclassdef
         bookkeeper = a.bookkeeper
 
         def getmdesc(bmeth):
@@ -100,14 +102,15 @@
         gfA_m = graphof(a, A.m.im_func)
         gfC_m = graphof(a, C.m.im_func)
 
-        assert famB_n.calltables == {(1, (), False): [{mdescB_n.funcdesc: gfB_n}] }
-        assert famA_m.calltables == {(1, (), False): [{mdescA_m.funcdesc: gfA_m, mdescC_m.funcdesc: gfC_m }] }
+        assert famB_n.calltables == {(1, (), False): [{mdescB_n.funcdesc: gfB_n}]}
+        assert famA_m.calltables == {(1, (), False): [
+            {mdescA_m.funcdesc: gfA_m, mdescC_m.funcdesc: gfC_m }]}
 
         mdescCinit = getmdesc(C().__init__)
         famCinit = mdescCinit.getcallfamily()
         gfCinit = graphof(a, C.__init__.im_func)
 
-        assert famCinit.calltables == {(1, (), False): [{mdescCinit.funcdesc: gfCinit}] }
+        assert famCinit.calltables == {(1, (), False): [{mdescCinit.funcdesc: gfCinit}]}
 
     def test_call_classes_with_noarg_init(self):
         class A:

From noreply at buildbot.pypy.org  Wed Apr 15 02:41:35 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Wed, 15 Apr 2015 02:41:35 +0200 (CEST)
Subject: [pypy-commit] pypy callfamily: turn nested function call_sites()
 into annotator method
Message-ID: <20150415004135.B8A9F1C0369@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: callfamily
Changeset: r76794:c118f115de13
Date: 2015-04-15 01:38 +0100
http://bitbucket.org/pypy/pypy/changeset/c118f115de13/

Log:	turn nested function call_sites() into annotator method

diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -290,6 +290,18 @@
         graph, block, index = position_key
         self.reflowpendingblock(graph, block)
 
+    def call_sites(self):
+        newblocks = self.added_blocks
+        if newblocks is None:
+            newblocks = self.annotated  # all of them
+        for block in newblocks:
+            for op in block.operations:
+                if op.opname in ('simple_call', 'call_args'):
+                    yield op
+
+                # some blocks are partially annotated
+                if op.result.annotation is None:
+                    break   # ignore the unannotated part
 
     #___ simplification (should be moved elsewhere?) _______
 
diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -92,21 +92,7 @@
         # one with a dummy position
         self.enter(None)
         try:
-            def call_sites():
-                newblocks = self.annotator.added_blocks
-                if newblocks is None:
-                    newblocks = self.annotator.annotated  # all of them
-                annotation = self.annotator.annotation
-                for block in newblocks:
-                    for op in block.operations:
-                        if op.opname in ('simple_call', 'call_args'):
-                            yield op
-
-                        # some blocks are partially annotated
-                        if annotation(op.result) is None:
-                            break   # ignore the unannotated part
-
-            for call_op in call_sites():
+            for call_op in self.annotator.call_sites():
                 self.consider_call_site(call_op)
 
             for pbc, args_s in self.emulated_pbc_calls.itervalues():

From noreply at buildbot.pypy.org  Wed Apr 15 09:44:42 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 15 Apr 2015 09:44:42 +0200 (CEST)
Subject: [pypy-commit] cffi default: issue #189: typedefs of function type
 (not function pointer)
Message-ID: <20150415074442.250FE1C01D6@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r1710:44de16be9be5
Date: 2015-04-15 09:43 +0200
http://bitbucket.org/cffi/cffi/changeset/44de16be9be5/

Log:	issue #189: typedefs of function type (not function pointer)

diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -266,7 +266,10 @@
             #
             if decl.name:
                 tp = self._get_type(node, partial_length_ok=True)
-                if self._is_constant_globalvar(node):
+                if tp.is_raw_function:
+                    tp = self._get_type_pointer(tp)
+                    self._declare('function ' + decl.name, tp)
+                elif self._is_constant_globalvar(node):
                     self._declare('constant ' + decl.name, tp)
                 else:
                     self._declare('variable ' + decl.name, tp)
diff --git a/testing/test_function.py b/testing/test_function.py
--- a/testing/test_function.py
+++ b/testing/test_function.py
@@ -292,7 +292,6 @@
         assert ffi.string(a) == b'4.4.4.4'
 
     def test_function_typedef(self):
-        py.test.skip("using really obscure C syntax")
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
             typedef double func_t(double);
diff --git a/testing/test_verify.py b/testing/test_verify.py
--- a/testing/test_verify.py
+++ b/testing/test_verify.py
@@ -1197,6 +1197,15 @@
     """)
     assert lib.foo_func(lib.BB) == lib.BB == 2
 
+def test_function_typedef():
+    ffi = FFI()
+    ffi.cdef("""
+        typedef double func_t(double);
+        func_t sin;
+    """)
+    lib = ffi.verify('#include ', libraries=lib_m)
+    assert lib.sin(1.23) == math.sin(1.23)
+
 def test_callback_calling_convention():
     py.test.skip("later")
     if sys.platform != 'win32':

From noreply at buildbot.pypy.org  Wed Apr 15 09:56:14 2015
From: noreply at buildbot.pypy.org (devin.jeanpierre)
Date: Wed, 15 Apr 2015 09:56:14 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Expand/clarify documentation of
	stm_become_inevitable.
Message-ID: <20150415075614.D8ED91C0498@cobra.cs.uni-duesseldorf.de>

Author: Devin Jeanpierre 
Branch: 
Changeset: r1755:c5225f44a522
Date: 2015-04-14 21:52 -0400
http://bitbucket.org/pypy/stmgc/changeset/c5225f44a522/

Log:	Expand/clarify documentation of stm_become_inevitable.

diff --git a/c7/stmgc.h b/c7/stmgc.h
--- a/c7/stmgc.h
+++ b/c7/stmgc.h
@@ -340,8 +340,6 @@
    returns: it jumps back to the stm_start_transaction(). */
 void stm_abort_transaction(void) __attribute__((noreturn));
 
-/* Turn the current transaction inevitable.
-   The stm_become_inevitable() itself may still abort. */
 #ifdef STM_NO_AUTOMATIC_SETJMP
 int stm_is_inevitable(void);
 #else
@@ -349,6 +347,10 @@
     return !rewind_jmp_armed(&STM_SEGMENT->running_thread->rjthread);
 }
 #endif
+
+/* Turn the current transaction inevitable.
+   stm_become_inevitable() itself may still abort the transaction instead
+   of returning. */
 static inline void stm_become_inevitable(stm_thread_local_t *tl,
                                          const char* msg) {
     assert(STM_SEGMENT->running_thread == tl);
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -341,6 +341,10 @@
     return !rewind_jmp_armed(&STM_SEGMENT->running_thread->rjthread);
 }
 #endif
+
+/* Turn the current transaction inevitable.
+   stm_become_inevitable() itself may still abort the transaction instead
+   of returning. */
 static inline void stm_become_inevitable(stm_thread_local_t *tl,
                                          const char* msg) {
     assert(STM_SEGMENT->running_thread == tl);

From noreply at buildbot.pypy.org  Wed Apr 15 12:26:08 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 15 Apr 2015 12:26:08 +0200 (CEST)
Subject: [pypy-commit] pypy default: 'default_factory' is actually a public
 read-write attribute on CPython.
Message-ID: <20150415102608.2563A1C0498@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76795:8374a14307bd
Date: 2015-04-15 12:26 +0200
http://bitbucket.org/pypy/pypy/changeset/8374a14307bd/

Log:	'default_factory' is actually a public read-write attribute on
	CPython.

diff --git a/pypy/module/_collections/app_defaultdict.py b/pypy/module/_collections/app_defaultdict.py
--- a/pypy/module/_collections/app_defaultdict.py
+++ b/pypy/module/_collections/app_defaultdict.py
@@ -11,7 +11,7 @@
 
 
 class defaultdict(dict):
-    __slots__ = ['__default_factory']
+    __slots__ = ['default_factory']
 
     def __init__(self, *args, **kwds):
         if len(args) > 0:
@@ -21,7 +21,7 @@
                 raise TypeError("first argument must be callable")
         else:
             default_factory = None
-        defaultdict.__default_factory.__set__(self, default_factory)
+        defaultdict.default_factory.__set__(self, default_factory)
         super(defaultdict, self).__init__(*args, **kwds)
 
     def __missing__(self, key):
@@ -34,13 +34,13 @@
             return "defaultdict(...)"
         try:
             recurse.add(id(self))
-            return "defaultdict(%s, %s)" % (self.__default_factory,
+            return "defaultdict(%s, %s)" % (self.default_factory,
                                             super(defaultdict, self).__repr__())
         finally:
             recurse.remove(id(self))
 
     def copy(self):
-        return type(self)(self.__default_factory, self)
+        return type(self)(self.default_factory, self)
 
     __copy__ = copy
 
@@ -56,5 +56,5 @@
 
            This API is used by pickle.py and copy.py.
         """
-        return (type(self), (self.__default_factory,), None, None,
+        return (type(self), (self.default_factory,), None, None,
                 defaultdict.iteritems(self))
diff --git a/pypy/module/_collections/interp_defaultdict.py b/pypy/module/_collections/interp_defaultdict.py
--- a/pypy/module/_collections/interp_defaultdict.py
+++ b/pypy/module/_collections/interp_defaultdict.py
@@ -4,8 +4,7 @@
     # An interp-level version of this method.  This is mostly only
     # useful because it can be executed atomically in the presence of
     # threads.
-    w_default_factory = space.getattr(w_self,
-            space.wrap('_defaultdict__default_factory'))
+    w_default_factory = space.getattr(w_self, space.wrap('default_factory'))
     if space.is_w(w_default_factory, space.w_None):
         raise OperationError(space.w_KeyError, space.newtuple([w_key]))
     w_value = space.call_function(w_default_factory)
diff --git a/pypy/module/_collections/test/test_defaultdict.py b/pypy/module/_collections/test/test_defaultdict.py
--- a/pypy/module/_collections/test/test_defaultdict.py
+++ b/pypy/module/_collections/test/test_defaultdict.py
@@ -68,3 +68,11 @@
         assert d['5'] == 0
         d['6'] += 3
         assert d['6'] == 3
+
+    def test_default_factory(self):
+        import _collections
+        f = lambda: 42
+        d = _collections.defaultdict(f)
+        assert d.default_factory is f
+        d.default_factory = lambda: 43
+        assert d['5'] == 43

From noreply at buildbot.pypy.org  Wed Apr 15 14:40:42 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 15 Apr 2015 14:40:42 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: ffi.new()
Message-ID: <20150415124042.225B91C00FC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1711:b4ccd7b6cbad
Date: 2015-04-15 14:41 +0200
http://bitbucket.org/cffi/cffi/changeset/b4ccd7b6cbad/

Log:	ffi.new()

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -2796,14 +2796,11 @@
     return (PyObject *)cd;
 }
 
-static PyObject *b_newp(PyObject *self, PyObject *args)
-{
-    CTypeDescrObject *ct, *ctitem;
+static PyObject *direct_newp(CTypeDescrObject *ct, PyObject *init)
+{
+    CTypeDescrObject *ctitem;
     CDataObject *cd;
-    PyObject *init = Py_None;
     Py_ssize_t dataoffset, datasize, explicitlength;
-    if (!PyArg_ParseTuple(args, "O!|O:newp", &CTypeDescr_Type, &ct, &init))
-        return NULL;
 
     explicitlength = -1;
     if (ct->ct_flags & CT_POINTER) {
@@ -2894,6 +2891,15 @@
     return (PyObject *)cd;
 }
 
+static PyObject *b_newp(PyObject *self, PyObject *args)
+{
+    CTypeDescrObject *ct;
+    PyObject *init = Py_None;
+    if (!PyArg_ParseTuple(args, "O!|O:newp", &CTypeDescr_Type, &ct, &init))
+        return NULL;
+    return direct_newp(ct, init);
+}
+
 static int
 _my_PyObject_AsBool(PyObject *ob)
 {
diff --git a/new/ffi_obj.c b/new/ffi_obj.c
--- a/new/ffi_obj.c
+++ b/new/ffi_obj.c
@@ -164,13 +164,10 @@
     return x;
 }
 
-#if 0
-static PyObject *ffi_new(ZefFFIObject *self, PyObject *args)
+static PyObject *ffi_new(FFIObject *self, PyObject *args)
 {
-    CTypeDescrObject *ct, *ctitem;
-    CDataObject *cd;
+    CTypeDescrObject *ct;
     PyObject *arg, *init = Py_None;
-    Py_ssize_t dataoffset, datasize, explicitlength;
     if (!PyArg_ParseTuple(args, "O|O:new", &arg, &init))
         return NULL;
 
@@ -178,76 +175,10 @@
     if (ct == NULL)
         return NULL;
 
-    explicitlength = -1;
-    if (ct->ct_flags & (CT_POINTER | CT_STRUCT | CT_UNION)) {
-        dataoffset = offsetof(CDataObject_own_nolength, alignment);
-        ctitem = (ct->ct_flags & CT_POINTER) ? ct->ct_itemdescr : ct;
-        datasize = ctitem->ct_size;
-        if (datasize < 0) {
-            PyErr_Format(PyExc_TypeError,
-                         "cannot instantiate ctype '%s' of unknown size",
-                         ctitem->ct_name);
-            return NULL;
-        }
-        if (ctitem->ct_flags & CT_PRIMITIVE_CHAR)
-            datasize *= 2;   /* forcefully add another character: a null */
-
-        if ((ctitem->ct_flags & CT_WITH_VAR_ARRAY) && init != Py_None) {
-            Py_ssize_t optvarsize = datasize;
-            if (convert_struct_from_object(NULL,ctitem, init, &optvarsize) < 0)
-                return NULL;
-            datasize = optvarsize;
-        }
-    }
-    else if (ct->ct_flags & CT_ARRAY) {
-        dataoffset = offsetof(CDataObject_own_nolength, alignment);
-        datasize = ct->ct_size;
-        if (datasize < 0) {
-            explicitlength = get_new_array_length(&init);
-            if (explicitlength < 0)
-                return NULL;
-            ctitem = ct->ct_itemdescr;
-            dataoffset = offsetof(CDataObject_own_length, alignment);
-            datasize = explicitlength * ctitem->ct_size;
-            if (explicitlength > 0 &&
-                    (datasize / explicitlength) != ctitem->ct_size) {
-                PyErr_SetString(PyExc_OverflowError,
-                                "array size would overflow a Py_ssize_t");
-                return NULL;
-            }
-        }
-    }
-    else if (ct->ct_flags & CT_PRIMITIVE_ANY) {
-        cd = _new_casted_primitive(ct);
-        datasize = ct->ct_size;
-        goto initialize_casted_primitive;
-    }
-    else {
-        PyErr_Format(PyExc_TypeError,
-                     "cannot create cdata '%s' objects", ct->ct_name);
-        return NULL;
-    }
-
-    cd = allocate_owning_object(dataoffset + datasize, ct);
-    if (cd == NULL)
-        return NULL;
-
-    cd->c_data = ((char *)cd) + dataoffset;
-    if (explicitlength >= 0)
-        ((CDataObject_own_length*)cd)->length = explicitlength;
-
- initialize_casted_primitive:
-    memset(cd->c_data, 0, datasize);
-    if (init != Py_None) {
-        if (convert_from_object(cd->c_data,
-              (ct->ct_flags & CT_POINTER) ? ct->ct_itemdescr : ct, init) < 0) {
-            Py_DECREF(cd);
-            return NULL;
-        }
-    }
-    return (PyObject *)cd;
+    return direct_newp(ct, init);
 }
 
+#if 0
 static PyObject *ffi_cast(ZefFFIObject *self, PyObject *args)
 {
     CTypeDescrObject *ct;
@@ -555,7 +486,9 @@
     {"getctype",      (PyCFunction)ffi_getctype,  METH_VARARGS},
     {"load_library",  (PyCFunction)ffi_load_library,METH_VARARGS|METH_KEYWORDS},
     {"offsetof",      (PyCFunction)ffi_offsetof,  METH_VARARGS},
+#endif
     {"new",           (PyCFunction)ffi_new,       METH_VARARGS},
+#if 0
     {"new_handle",    (PyCFunction)ffi_new_handle,METH_O},
 #endif
     {"sizeof",        (PyCFunction)ffi_sizeof,    METH_O},
diff --git a/new/test_ffi_obj.py b/new/test_ffi_obj.py
new file mode 100644
--- /dev/null
+++ b/new/test_ffi_obj.py
@@ -0,0 +1,7 @@
+import _cffi1_backend
+
+def test_ffi_new():
+    ffi = _cffi1_backend.FFI()
+    p = ffi.new("int *")
+    p[0] = -42
+    assert p[0] == -42

From noreply at buildbot.pypy.org  Wed Apr 15 15:14:58 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 15 Apr 2015 15:14:58 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: subclassing of the FFI class
Message-ID: <20150415131458.2849D1C0362@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1712:dc08ca1cf470
Date: 2015-04-15 15:15 +0200
http://bitbucket.org/cffi/cffi/changeset/dc08ca1cf470/

Log:	subclassing of the FFI class

diff --git a/new/cffi1_module.c b/new/cffi1_module.c
--- a/new/cffi1_module.c
+++ b/new/cffi1_module.c
@@ -46,7 +46,8 @@
     if (m == NULL)
         return -1;
 
-    FFIObject *ffi = ffi_internal_new(ctx);
+    FFIObject *ffi = ffi_internal_new(NULL, ctx);
+    Py_XINCREF(ffi);    /* make the ffi object really immortal */
     if (ffi == NULL || PyModule_AddObject(m, "ffi", (PyObject *)ffi) < 0)
         return -1;
 
diff --git a/new/ffi_obj.c b/new/ffi_obj.c
--- a/new/ffi_obj.c
+++ b/new/ffi_obj.c
@@ -26,13 +26,22 @@
     _cffi_opcode_t internal_output[FFI_COMPLEXITY_OUTPUT];
 };
 
-static FFIObject *ffi_internal_new(const struct _cffi_type_context_s *ctx)
+static FFIObject *ffi_internal_new(PyTypeObject *ffitype,
+                                   const struct _cffi_type_context_s *ctx)
 {
     PyObject *dict = PyDict_New();
     if (dict == NULL)
         return NULL;
 
-    FFIObject *ffi = PyObject_GC_New(FFIObject, &FFI_Type);
+    FFIObject *ffi;
+    if (ffitype == NULL) {
+        ffi = (FFIObject *)PyObject_New(FFIObject, &FFI_Type);
+        /* we don't call PyObject_GC_Track() here: from _cffi_init_module()
+           it is not needed, because in this case the ffi object is immortal */
+    }
+    else {
+        ffi = (FFIObject *)ffitype->tp_alloc(ffitype, 0);
+    }
     if (ffi == NULL) {
         Py_DECREF(dict);
         return NULL;
@@ -42,8 +51,6 @@
     ffi->info.ctx = ctx;
     ffi->info.output = ffi->internal_output;
     ffi->info.output_size = FFI_COMPLEXITY_OUTPUT;
-
-    PyObject_GC_Track(ffi);
     return ffi;
 }
 
@@ -52,7 +59,23 @@
     PyObject_GC_UnTrack(ffi);
     Py_DECREF(ffi->types_dict);
     Py_XDECREF(ffi->gc_wrefs);
-    PyObject_GC_Del(ffi);
+
+    {
+        const void *mem[] = {ffi->info.ctx->types,
+                             ffi->info.ctx->globals,
+                             ffi->info.ctx->constants,
+                             ffi->info.ctx->structs_unions,
+                             ffi->info.ctx->fields,
+                             ffi->info.ctx->enums,
+                             ffi->info.ctx->typenames,
+                             ffi->info.ctx};
+        int i;
+        for (i = 0; i < sizeof(mem) / sizeof(*mem); i++) {
+            if (mem[i] != NULL)
+                PyMem_Free((void *)mem[i]);
+        }
+    }
+    Py_TYPE(ffi)->tp_free((PyObject *)ffi);
 }
 
 static int ffi_traverse(FFIObject *ffi, visitproc visit, void *arg)
@@ -65,13 +88,30 @@
 static PyObject *ffiobj_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 {
     /* user-facing initialization code, for explicit FFI() calls */
-    static const struct _cffi_type_context_s empty_ctx = { 0 };
+    struct _cffi_type_context_s *ctx;
+    PyObject *result;
 
+    ctx = PyMem_Malloc(sizeof(struct _cffi_type_context_s));
+    if (ctx == NULL) {
+        PyErr_NoMemory();
+        return NULL;
+    }
+    memset(ctx, 0, sizeof(struct _cffi_type_context_s));
+
+    result = (PyObject *)ffi_internal_new(type, ctx);
+    if (result == NULL) {
+        PyMem_Free(ctx);
+        return NULL;
+    }
+    return result;
+}
+
+static int ffiobj_init(PyObject *self, PyObject *args, PyObject *kwds)
+{
     char *keywords[] = {NULL};
     if (!PyArg_ParseTupleAndKeywords(args, kwds, ":FFI", keywords))
-        return NULL;
-
-    return (PyObject *)ffi_internal_new(&empty_ctx);
+        return -1;
+    return 0;
 }
 
 #define ACCEPT_STRING   1
@@ -519,7 +559,8 @@
     PyObject_GenericGetAttr,                    /* tp_getattro */
     0,                                          /* tp_setattro */
     0,                                          /* tp_as_buffer */
-    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,    /* tp_flags */
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
+        Py_TPFLAGS_BASETYPE,                    /* tp_flags */
     0,                                          /* tp_doc */
     (traverseproc)ffi_traverse,                 /* tp_traverse */
     0,                                          /* tp_clear */
@@ -535,7 +576,7 @@
     0,                                          /* tp_descr_get */
     0,                                          /* tp_descr_set */
     0,                                          /* tp_dictoffset */
-    0,                                          /* tp_init */
+    ffiobj_init,                                /* tp_init */
     0,                                          /* tp_alloc */
     ffiobj_new,                                 /* tp_new */
     PyObject_GC_Del,                            /* tp_free */
diff --git a/new/test_ffi_obj.py b/new/test_ffi_obj.py
--- a/new/test_ffi_obj.py
+++ b/new/test_ffi_obj.py
@@ -1,7 +1,21 @@
+import py
 import _cffi1_backend
 
+
 def test_ffi_new():
     ffi = _cffi1_backend.FFI()
     p = ffi.new("int *")
     p[0] = -42
     assert p[0] == -42
+
+def test_ffi_subclass():
+    class FOO(_cffi1_backend.FFI):
+        def __init__(self, x):
+            self.x = x
+    foo = FOO(42)
+    assert foo.x == 42
+    p = foo.new("int *")
+    assert p[0] == 0
+
+def test_ffi_no_argument():
+    py.test.raises(TypeError, _cffi1_backend.FFI, 42)

From noreply at buildbot.pypy.org  Wed Apr 15 15:55:53 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Wed, 15 Apr 2015 15:55:53 +0200 (CEST)
Subject: [pypy-commit] pypy callfamily: Simplify row_to_consider() by not
 calling complicated method desc.pycall()
Message-ID: <20150415135553.AB32F1C00FC@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: callfamily
Changeset: r76796:d5e80ca32789
Date: 2015-04-15 14:55 +0100
http://bitbucket.org/pypy/pypy/changeset/d5e80ca32789/

Log:	Simplify row_to_consider() by not calling complicated method
	desc.pycall()

diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py
--- a/rpython/annotator/description.py
+++ b/rpython/annotator/description.py
@@ -320,6 +320,10 @@
         result = unionof(result, s_previous_result)
         return result
 
+    def get_graph(self, args, op):
+        inputs_s = self.parse_arguments(args)
+        return self.specialize(inputs_s, op)
+
     def get_call_parameters(self, args_s):
         args = simple_args(args_s)
         inputcells = self.parse_arguments(args)
@@ -372,11 +376,9 @@
         # see comments in CallFamily
         row = {}
         for desc in descs:
-            def enlist(graph, ignore):
-                row[desc.rowkey()] = graph
-                return s_ImpossibleValue   # meaningless
-            desc.pycall(enlist, args, s_ImpossibleValue, op)
-            assert row
+            graph = desc.get_graph(args, op)
+            assert isinstance(graph, FunctionGraph)
+            row[desc.rowkey()] = graph
         return row
 
     def get_s_signatures(self, shape):
@@ -893,13 +895,20 @@
     def getuniquegraph(self):
         return self.funcdesc.getuniquegraph()
 
-    def pycall(self, schedule, args, s_previous_result, op=None):
+    def func_args(self, args):
         from rpython.annotator.model import SomeInstance
         if self.selfclassdef is None:
             raise Exception("calling %r" % (self,))
         s_instance = SomeInstance(self.selfclassdef, flags=self.flags)
-        args = args.prepend(s_instance)
-        return self.funcdesc.pycall(schedule, args, s_previous_result, op)
+        return args.prepend(s_instance)
+
+    def pycall(self, schedule, args, s_previous_result, op=None):
+        func_args = self.func_args(args)
+        return self.funcdesc.pycall(schedule, func_args, s_previous_result, op)
+
+    def get_graph(self, args, op):
+        func_args = self.func_args(args)
+        return self.funcdesc.get_graph(func_args, op)
 
     def bind_under(self, classdef, name):
         self.bookkeeper.warning("rebinding an already bound %r" % (self,))
@@ -1068,11 +1077,19 @@
         return '' % (self.funcdesc,
                                                   self.frozendesc)
 
-    def pycall(self, schedule, args, s_previous_result, op=None):
+    def func_args(self, args):
         from rpython.annotator.model import SomePBC
         s_self = SomePBC([self.frozendesc])
-        args = args.prepend(s_self)
-        return self.funcdesc.pycall(schedule, args, s_previous_result, op)
+        return args.prepend(s_self)
+
+    def pycall(self, schedule, args, s_previous_result, op=None):
+        func_args = self.func_args(args)
+        return self.funcdesc.pycall(schedule, func_args, s_previous_result, op)
+
+    def get_graph(self, args, op):
+        func_args = self.func_args(args)
+        return self.funcdesc.get_graph(func_args, op)
+
 
     @staticmethod
     def consider_call_site(descs, args, s_result, op):

From noreply at buildbot.pypy.org  Wed Apr 15 16:31:49 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Wed, 15 Apr 2015 16:31:49 +0200 (CEST)
Subject: [pypy-commit] pypy callfamily: kill one crazy callback
Message-ID: <20150415143149.DA2C51C02A3@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: callfamily
Changeset: r76797:da3f97e3f9ab
Date: 2015-04-15 15:32 +0100
http://bitbucket.org/pypy/pypy/changeset/da3f97e3f9ab/

Log:	kill one crazy callback

diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -498,12 +498,9 @@
             op = None
             s_previous_result = s_ImpossibleValue
 
-        def schedule(graph, inputcells):
-            return self.annotator.recursivecall(graph, whence, inputcells)
-
         results = []
         for desc in pbc.descriptions:
-            results.append(desc.pycall(schedule, args, s_previous_result, op))
+            results.append(desc.pycall(whence, args, s_previous_result, op))
         s_result = unionof(*results)
         return s_result
 
diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py
--- a/rpython/annotator/description.py
+++ b/rpython/annotator/description.py
@@ -295,22 +295,23 @@
         else:
             return self.specializer(self, inputcells)
 
-    def pycall(self, schedule, args, s_previous_result, op=None):
+    def pycall(self, whence, args, s_previous_result, op=None):
         inputcells = self.parse_arguments(args)
         result = self.specialize(inputcells, op)
         if isinstance(result, FunctionGraph):
             graph = result         # common case
+            annotator = self.bookkeeper.annotator
             # if that graph has a different signature, we need to re-parse
             # the arguments.
             # recreate the args object because inputcells may have been changed
             new_args = args.unmatch_signature(self.signature, inputcells)
             inputcells = self.parse_arguments(new_args, graph)
-            result = schedule(graph, inputcells)
+            result = annotator.recursivecall(graph, whence, inputcells)
             signature = getattr(self.pyobj, '_signature_', None)
             if signature:
                 sigresult = enforce_signature_return(self, signature[1], result)
                 if sigresult is not None:
-                    self.bookkeeper.annotator.addpendingblock(
+                    annotator.addpendingblock(
                         graph, graph.returnblock, [sigresult])
                     result = sigresult
         # Some specializations may break the invariant of returning
@@ -627,7 +628,7 @@
                             "specialization" % (self.name,))
         return self.getclassdef(None)
 
-    def pycall(self, schedule, args, s_previous_result, op=None):
+    def pycall(self, whence, args, s_previous_result, op=None):
         from rpython.annotator.model import SomeInstance, SomeImpossibleValue
         if self.specialize:
             if self.specialize == 'specialize:ctr_location':
@@ -902,9 +903,9 @@
         s_instance = SomeInstance(self.selfclassdef, flags=self.flags)
         return args.prepend(s_instance)
 
-    def pycall(self, schedule, args, s_previous_result, op=None):
+    def pycall(self, whence, args, s_previous_result, op=None):
         func_args = self.func_args(args)
-        return self.funcdesc.pycall(schedule, func_args, s_previous_result, op)
+        return self.funcdesc.pycall(whence, func_args, s_previous_result, op)
 
     def get_graph(self, args, op):
         func_args = self.func_args(args)
@@ -1082,9 +1083,9 @@
         s_self = SomePBC([self.frozendesc])
         return args.prepend(s_self)
 
-    def pycall(self, schedule, args, s_previous_result, op=None):
+    def pycall(self, whence, args, s_previous_result, op=None):
         func_args = self.func_args(args)
-        return self.funcdesc.pycall(schedule, func_args, s_previous_result, op)
+        return self.funcdesc.pycall(whence, func_args, s_previous_result, op)
 
     def get_graph(self, args, op):
         func_args = self.func_args(args)

From noreply at buildbot.pypy.org  Wed Apr 15 18:12:53 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 15 Apr 2015 18:12:53 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Starting working on the "recompiler"
Message-ID: <20150415161253.05A7E1C00FC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1713:0b449baa47ff
Date: 2015-04-15 18:13 +0200
http://bitbucket.org/cffi/cffi/changeset/0b449baa47ff/

Log:	Starting working on the "recompiler"

diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -1,5 +1,6 @@
 import sys, types
 from .lock import allocate_lock
+import _cffi1_backend
 
 try:
     callable
@@ -27,7 +28,7 @@
         return '%s%s' % (line, self.args[0])
 
 
-class FFI(object):
+class FFI(_cffi1_backend.FFI):
     r'''
     The main top-level class that you instantiate once, or once per module.
 
@@ -45,21 +46,16 @@
         C.printf("hello, %s!\n", ffi.new("char[]", "world"))
     '''
 
-    def __init__(self, backend=None):
+    def __init__(self):
         """Create an FFI instance.  The 'backend' argument is used to
         select a non-default backend, mostly for tests.
         """
         from . import cparser, model
-        if backend is None:
-            # You need PyPy (>= 2.0 beta), or a CPython (>= 2.6) with
-            # _cffi_backend.so compiled.
-            import _cffi_backend as backend
-            from . import __version__
-            assert backend.__version__ == __version__, \
-               "version mismatch, %s != %s" % (backend.__version__, __version__)
-            # (If you insist you can also try to pass the option
-            # 'backend=backend_ctypes.CTypesBackend()', but don't
-            # rely on it!  It's probably not going to work well.)
+        from . import __version__
+
+        backend = _cffi1_backend
+        assert backend.__version__ == __version__, \
+            "version mismatch, %s != %s" % (backend.__version__, __version__)
 
         self._backend = backend
         self._lock = allocate_lock()
@@ -80,15 +76,6 @@
         with self._lock:
             self.BVoidP = self._get_cached_btype(model.voidp_type)
             self.BCharA = self._get_cached_btype(model.char_array_type)
-        if isinstance(backend, types.ModuleType):
-            # _cffi_backend: attach these constants to the class
-            if not hasattr(FFI, 'NULL'):
-                FFI.NULL = self.cast(self.BVoidP, 0)
-                FFI.CData, FFI.CType = backend._get_types()
-        else:
-            # ctypes backend: attach these constants to the instance
-            self.NULL = self.cast(self.BVoidP, 0)
-            self.CData, self.CType = backend._get_types()
 
     def cdef(self, csource, override=False, packed=False):
         """Parse the given C source.  This registers all declared functions,
@@ -203,7 +190,7 @@
             cdecl = self._typeof(cdecl)
         return self._typeoffsetof(cdecl, *fields_or_indexes)[1]
 
-    def new(self, cdecl, init=None):
+    def XXXnew(self, cdecl, init=None):
         """Allocate an instance according to the specified C type and
         return a pointer to it.  The specified C type must be either a
         pointer or an array: ``new('X *')`` allocates an X and returns
diff --git a/cffi/model.py b/cffi/model.py
--- a/cffi/model.py
+++ b/cffi/model.py
@@ -169,6 +169,9 @@
         return global_cache(self, ffi, 'new_function_type',
                             tuple(args), result, self.ellipsis)
 
+    def as_raw_function(self):
+        return RawFunctionType(self.args, self.result, self.ellipsis)
+
 
 class PointerType(BaseType):
     _attrs_ = ('totype',)
diff --git a/new/cffi1 b/new/cffi1
new file mode 120000
--- /dev/null
+++ b/new/cffi1
@@ -0,0 +1,1 @@
+../cffi
\ No newline at end of file
diff --git a/new/cffi1_module.c b/new/cffi1_module.c
--- a/new/cffi1_module.c
+++ b/new/cffi1_module.c
@@ -28,6 +28,9 @@
     if (PyDict_SetItemString(FFI_Type.tp_dict, "CType",
                              (PyObject *)&CTypeDescr_Type) < 0)
         return -1;
+    if (PyDict_SetItemString(FFI_Type.tp_dict, "CData",
+                             (PyObject *)&CData_Type) < 0)
+        return -1;
 
     Py_INCREF(&FFI_Type);
     if (PyModule_AddObject(m, "FFI", (PyObject *)&FFI_Type) < 0)
diff --git a/new/cffi_opcode.py b/new/cffi_opcode.py
new file mode 100644
--- /dev/null
+++ b/new/cffi_opcode.py
@@ -0,0 +1,53 @@
+
+class CffiOp(object):
+    def __init__(self, op, arg):
+        self.op = op
+        self.arg = arg
+    def as_int(self):
+        return self.op | (self.arg << 8)
+    def __str__(self):
+        classname = CLASS_NAME.get(self.op, self.op)
+        return '(%s %d)' % (classname, self.arg)
+
+OP_PRIMITIVE       = 1
+OP_POINTER         = 3
+OP_ARRAY           = 5
+OP_OPEN_ARRAY      = 7
+OP_STRUCT_UNION    = 9
+OP_ENUM            = 11
+OP_TYPENAME        = 13
+OP_FUNCTION        = 15
+OP_FUNCTION_END    = 17
+OP_NOOP            = 19
+OP_BITFIELD        = 21
+OP_CPYTHON_BLTN_V  = 23   # varargs
+OP_CPYTHON_BLTN_N  = 25   # noargs
+OP_CPYTHON_BLTN_O  = 27   # O  (i.e. a single arg)
+
+PRIM_VOID          = 0
+PRIM_BOOL          = 1
+PRIM_CHAR          = 2
+PRIM_SCHAR         = 3
+PRIM_UCHAR         = 4
+PRIM_SHORT         = 5
+PRIM_USHORT        = 6
+PRIM_INT           = 7
+PRIM_UINT          = 8
+PRIM_LONG          = 9
+PRIM_ULONG         = 10
+PRIM_LONGLONG      = 11
+PRIM_ULONGLONG     = 12
+PRIM_FLOAT         = 13
+PRIM_DOUBLE        = 14
+PRIM_LONGDOUBLE    = 15
+
+PRIMITIVE_TO_INDEX = {
+    'int':    PRIM_INT,
+    'float':  PRIM_FLOAT,
+    'double': PRIM_DOUBLE,
+    }
+
+CLASS_NAME = {}
+for _name, _value in globals().items():
+    if _name.startswith('OP_') and isinstance(_value, int):
+        CLASS_NAME[_value] = _name[3:]
diff --git a/new/lib_obj.c b/new/lib_obj.c
--- a/new/lib_obj.c
+++ b/new/lib_obj.c
@@ -1,22 +1,14 @@
 
-/* A Lib object is what is returned by any of:
-
-   - the "lib" attribute of a C extension module originally created by
-     recompile()
-
-   - ffi.dlopen()
-
-   - ffi.verify()
+/* A Lib object is what is in the "lib" attribute of a C extension
+   module originally created by recompile().
 
    A Lib object is special in the sense that it has a custom
    __getattr__ which returns C globals, functions and constants.  It
-   raises AttributeError for anything else, like '__class__'.
+   raises AttributeError for anything else, even attrs like '__class__'.
 
    A Lib object has got a reference to the _cffi_type_context_s
    structure, which is used to create lazily the objects returned by
-   __getattr__.  For a dlopen()ed Lib object, all the 'address' fields
-   in _cffi_global_s are NULL, and instead dlsym() is used lazily on
-   the l_dl_lib.
+   __getattr__.
 */
 
 struct CPyExtFunc_s {
@@ -29,17 +21,13 @@
     PyObject_HEAD
     const struct _cffi_type_context_s *l_ctx;  /* ctx object */
     PyObject *l_dict;           /* content, built lazily */
-    void *l_dl_lib;             /* the result of 'dlopen()', or NULL */
     PyObject *l_libname;        /* some string that gives the name of the lib */
 };
 
 #define LibObject_Check(ob)  ((Py_TYPE(ob) == &Lib_Type))
 
-static int lib_close(LibObject *lib);    /* forward */
-
 static void lib_dealloc(LibObject *lib)
 {
-    (void)lib_close(lib);
     Py_DECREF(lib->l_dict);
     Py_DECREF(lib->l_libname);
     PyObject_Del(lib);
@@ -228,33 +216,6 @@
     offsetof(LibObject, l_dict),                /* tp_dictoffset */
 };
 
-
-static void lib_dlerror(LibObject *lib)
-{
-    char *error = dlerror();
-    if (error == NULL)
-        error = "(no error reported)";
-    PyErr_Format(PyExc_OSError, "%s: %s", PyText_AS_UTF8(lib->l_libname),
-                 error);
-}
-
-static int lib_close(LibObject *lib)
-{
-    void *dll;
-    lib->l_ctx = NULL;
-    PyDict_Clear(lib->l_dict);
-
-    dll = lib->l_dl_lib;
-    if (dll != NULL) {
-        lib->l_dl_lib = NULL;
-        if (dlclose(dll) != 0) {
-            lib_dlerror(lib);
-            return -1;
-        }
-    }
-    return 0;
-}
-
 static LibObject *lib_internal_new(const struct _cffi_type_context_s *ctx,
                                    char *module_name)
 {
@@ -275,7 +236,6 @@
 
     lib->l_ctx = ctx;
     lib->l_dict = dict;
-    lib->l_dl_lib = NULL;
     lib->l_libname = libname;
     return lib;
 }
diff --git a/new/recompiler.py b/new/recompiler.py
new file mode 100644
--- /dev/null
+++ b/new/recompiler.py
@@ -0,0 +1,132 @@
+import os
+from cffi1 import ffiplatform, model
+from cffi_opcode import *
+
+
+class Recompiler:
+
+    def __init__(self, ffi):
+        self.ffi = ffi
+
+    def collect_type_table(self):
+        self._typesdict = {}
+        self._generate('collecttype')
+        #
+        all_decls = sorted(self._typesdict, key=str)
+        #
+        # prepare all FUNCTION bytecode sequences first
+        self.cffi_types = []
+        for tp in all_decls:
+            if tp.is_raw_function:
+                assert self._typesdict[tp] is None
+                self._typesdict[tp] = len(self.cffi_types)
+                self.cffi_types.append(tp)     # placeholder
+                for tp1 in tp.args:
+                    assert isinstance(tp1, (model.VoidType,
+                                            model.PrimitiveType,
+                                            model.PointerType,
+                                            model.StructOrUnionOrEnum,
+                                            model.FunctionPtrType))
+                    if self._typesdict[tp1] is None:
+                        self._typesdict[tp1] = len(self.cffi_types)
+                    self.cffi_types.append(tp1)   # placeholder
+                self.cffi_types.append('END')     # placeholder
+        #
+        # prepare all OTHER bytecode sequences
+        for tp in all_decls:
+            if not tp.is_raw_function and self._typesdict[tp] is None:
+                self._typesdict[tp] = len(self.cffi_types)
+                self.cffi_types.append(tp)        # placeholder
+                if tp.is_array_type and tp.length is not None:
+                    self.cffi_types.append('LEN') # placeholder
+        assert None not in self._typesdict.values()
+        #
+        # emit all bytecode sequences now
+        for tp in all_decls:
+            method = getattr(self, '_emit_bytecode_' + tp.__class__.__name__)
+            method(tp, self._typesdict[tp])
+        #
+        # consistency check
+        for op in self.cffi_types:
+            assert isinstance(op, CffiOp)
+
+    def _do_collect_type(self, tp):
+        if not isinstance(tp, model.BaseTypeByIdentity):
+            if isinstance(tp, tuple):
+                for x in tp:
+                    self._do_collect_type(x)
+            return
+        if tp not in self._typesdict:
+            self._typesdict[tp] = None
+            if isinstance(tp, model.FunctionPtrType):
+                self._do_collect_type(tp.as_raw_function())
+            else:
+                for _, x in tp._get_items():
+                    self._do_collect_type(x)
+
+    def _get_declarations(self):
+        return sorted(self.ffi._parser._declarations.items())
+
+    def _generate(self, step_name):
+        for name, tp in self._get_declarations():
+            kind, realname = name.split(' ', 1)
+            try:
+                method = getattr(self, '_generate_cpy_%s_%s' % (kind,
+                                                                step_name))
+            except AttributeError:
+                raise ffiplatform.VerificationError(
+                    "not implemented in verify(): %r" % name)
+            try:
+                method(tp, realname)
+            except Exception as e:
+                model.attach_exception_info(e, name)
+                raise
+
+    def _generate_cpy_function_collecttype(self, tp, name):
+        self._do_collect_type(tp.as_raw_function())
+
+    def _emit_bytecode_PrimitiveType(self, tp, index):
+        prim_index = PRIMITIVE_TO_INDEX[tp.name]
+        self.cffi_types[index] = CffiOp(OP_PRIMITIVE, prim_index)
+
+    def _emit_bytecode_RawFunctionType(self, tp, index):
+        self.cffi_types[index] = CffiOp(OP_FUNCTION, self._typesdict[tp.result])
+        index += 1
+        for tp1 in tp.args:
+            realindex = self._typesdict[tp1]
+            if index != realindex:
+                if isinstance(tp1, model.PrimitiveType):
+                    self._emit_bytecode_PrimitiveType(tp1, index)
+                else:
+                    self.cffi_types[index] = CffiOp(OP_NOOP, realindex)
+            index += 1
+        self.cffi_types[index] = CffiOp(OP_FUNCTION_END, tp.ellipsis)
+
+    def _emit_bytecode_PointerType(self, tp, index):
+        self.cffi_types[index] = CffiOp(OP_POINTER, self._typesdict[tp.totype])
+
+    # ----------
+
+    def _prnt(self, what=''):
+        self._f.write(what + '\n')
+
+    def write_source_to_f(self, f, preamble):
+        self._f = f
+        prnt = self._prnt
+        # first copy some standard set of lines that are mostly '#define'
+        filename = os.path.join(os.path.dirname(__file__), '_cffi_include.h')
+        with open(filename, 'r') as g:
+            prnt(g.read())
+        prnt('/************************************************************/')
+        prnt()
+        # then paste the C source given by the user, verbatim.
+        prnt(preamble)
+        prnt()
+        #...
+        
+
+def make_c_source(ffi, target_c_file, preamble):
+    recompiler = Recompiler(ffi)
+    recompiler.collect_type_table()
+    with open(target_c_file, 'w') as f:
+        recompiler.write_source_to_f(f, preamble)
diff --git a/new/test_dlopen.py b/new/test_dlopen.py
new file mode 100644
--- /dev/null
+++ b/new/test_dlopen.py
@@ -0,0 +1,10 @@
+from cffi1 import FFI
+import math
+
+
+def test_math_sin():
+    ffi = FFI()
+    ffi.cdef("double sin(double);")
+    m = ffi.dlopen('m')
+    x = m.sin(1.23)
+    assert x == math.sin(1.23)
diff --git a/new/test_recompiler.py b/new/test_recompiler.py
new file mode 100644
--- /dev/null
+++ b/new/test_recompiler.py
@@ -0,0 +1,42 @@
+from recompiler import Recompiler, make_c_source
+from cffi1 import FFI
+from udir import udir
+
+
+def check_type_table(input, expected_output):
+    ffi = FFI()
+    ffi.cdef(input)
+    recompiler = Recompiler(ffi)
+    recompiler.collect_type_table()
+    assert ''.join(map(str, recompiler.cffi_types)) == expected_output
+
+def test_type_table_func():
+    check_type_table("double sin(double);",
+                     "(FUNCTION 1)(PRIMITIVE 14)(FUNCTION_END 0)")
+    check_type_table("float sin(double);",
+                     "(FUNCTION 3)(PRIMITIVE 14)(FUNCTION_END 0)(PRIMITIVE 13)")
+    check_type_table("float sin(void);",
+                     "(FUNCTION 2)(FUNCTION_END 0)(PRIMITIVE 13)")
+    check_type_table("double sin(float); double cos(float);",
+                     "(FUNCTION 3)(PRIMITIVE 13)(FUNCTION_END 0)(PRIMITIVE 14)")
+    check_type_table("double sin(float); double cos(double);",
+                     "(FUNCTION 1)(PRIMITIVE 14)(FUNCTION_END 0)"   # cos
+                     "(FUNCTION 1)(PRIMITIVE 13)(FUNCTION_END 0)")  # sin
+    check_type_table("float sin(double); float cos(float);",
+                     "(FUNCTION 4)(PRIMITIVE 14)(FUNCTION_END 0)"   # sin
+                     "(FUNCTION 4)(PRIMITIVE 13)(FUNCTION_END 0)")  # cos
+
+def test_use_noop_for_repeated_args():
+    check_type_table("double sin(double *, double *);",
+                     "(FUNCTION 4)(POINTER 4)(NOOP 1)(FUNCTION_END 0)"
+                     "(PRIMITIVE 14)")
+
+def test_dont_use_noop_for_primitives():
+    check_type_table("double sin(double, double);",
+                     "(FUNCTION 1)(PRIMITIVE 14)(PRIMITIVE 14)(FUNCTION_END 0)")
+
+
+def test_math_sin():
+    ffi = FFI()
+    ffi.cdef("double sin(double);")
+    make_c_source(ffi, str(udir.join('math_sin.c')), '#include ')
diff --git a/testing/udir.py b/new/udir.py
copy from testing/udir.py
copy to new/udir.py

From noreply at buildbot.pypy.org  Wed Apr 15 18:51:58 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 15 Apr 2015 18:51:58 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Copy more code from vengine_cpy,
 and generate the remaining parts of the
Message-ID: <20150415165158.320041C02A3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1714:66428c0f8a54
Date: 2015-04-15 18:52 +0200
http://bitbucket.org/cffi/cffi/changeset/66428c0f8a54/

Log:	Copy more code from vengine_cpy, and generate the remaining parts of
	the example "manual.c"

diff --git a/new/cffi_opcode.py b/new/cffi_opcode.py
--- a/new/cffi_opcode.py
+++ b/new/cffi_opcode.py
@@ -3,8 +3,9 @@
     def __init__(self, op, arg):
         self.op = op
         self.arg = arg
-    def as_int(self):
-        return self.op | (self.arg << 8)
+    def as_c_expr(self):
+        classname = CLASS_NAME[self.op]
+        return '_CFFI_OP(_CFFI_OP_%s, %d)' % (classname, self.arg)
     def __str__(self):
         classname = CLASS_NAME.get(self.op, self.op)
         return '(%s %d)' % (classname, self.arg)
diff --git a/new/recompiler.py b/new/recompiler.py
--- a/new/recompiler.py
+++ b/new/recompiler.py
@@ -5,8 +5,9 @@
 
 class Recompiler:
 
-    def __init__(self, ffi):
+    def __init__(self, ffi, module_name):
         self.ffi = ffi
+        self.module_name = module_name
 
     def collect_type_table(self):
         self._typesdict = {}
@@ -113,20 +114,272 @@
     def write_source_to_f(self, f, preamble):
         self._f = f
         prnt = self._prnt
-        # first copy some standard set of lines that are mostly '#define'
-        filename = os.path.join(os.path.dirname(__file__), '_cffi_include.h')
-        with open(filename, 'r') as g:
-            prnt(g.read())
+        #
+        # first the '#include'
+        prnt('#include "_cffi_include.h"')
+        #
+        # then paste the C source given by the user, verbatim.
         prnt('/************************************************************/')
         prnt()
-        # then paste the C source given by the user, verbatim.
         prnt(preamble)
         prnt()
-        #...
-        
+        prnt('/************************************************************/')
+        prnt()
+        #
+        # the declaration of '_cffi_types'
+        prnt('static void *_cffi_types[] = {')
+        for op in self.cffi_types:
+            prnt('  %s,' % (op.as_c_expr(),))
+        if not self.cffi_types:
+            prnt('  0')
+        prnt('};')
+        prnt()
+        #
+        # call generate_cpy_xxx_decl(), for every xxx found from
+        # ffi._parser._declarations.  This generates all the functions.
+        self._generate("decl")
+        #
+        # the declaration of '_cffi_globals'
+        self._lst = []
+        self._generate("global")
+        num_globals = len(self._lst)
+        if num_globals > 0:
+            self._lst.sort()  # sort by name, which is at the start of each line
+            prnt('static const struct _cffi_global_s _cffi_globals[] = {')
+            for line in self._lst:
+                prnt(line)
+            prnt('};')
+            prnt()
+        #
+        # XXX
+        num_constants = 0
+        num_structs_unions = 0
+        num_enums = 0
+        num_typenames = 0
+        #
+        # the declaration of '_cffi_type_context'
+        prnt('static const struct _cffi_type_context_s _cffi_type_context = {')
+        prnt('  _cffi_types,')
+        if num_globals > 0:
+            prnt('  _cffi_globals,')
+        else:
+            prnt('  NULL,  /* no globals */')
+        if num_constants > 0:
+            prnt('  _cffi_constants,')
+        else:
+            prnt('  NULL,  /* no constants */')
+        if num_structs_unions > 0:
+            prnt('  _cffi_structs_unions,')
+            prnt('  _cffi_fields,')
+        else:
+            prnt('  NULL,  /* no structs */')
+            prnt('  NULL,  /* no fields */')
+        if num_enums > 0:
+            prnt('  _cffi_enums,')
+        else:
+            prnt('  NULL,  /* no enums */')
+        if num_typenames > 0:
+            prnt('  _cffi_typenames,')
+        else:
+            prnt('  NULL,  /* no typenames */')
+        prnt('  %d,  /* num_globals */' % num_globals)
+        prnt('  %d,  /* num_constants */' % num_constants)
+        prnt('  %d,  /* num_structs_unions */' % num_structs_unions)
+        prnt('  %d,  /* num_enums */' % num_enums)
+        prnt('  %d,  /* num_typenames */' % num_typenames)
+        prnt('};')
+        prnt()
+        #
+        # the init function, loading _cffi1_backend and calling a method there
+        prnt('PyMODINIT_FUNC')
+        prnt('init%s(void)' % (self.module_name,))
+        prnt('{')
+        prnt('  if (_cffi_init() < 0)')
+        prnt('    return;')
+        prnt('  _cffi_init_module("%s", &_cffi_type_context);' % (
+            self.module_name,))
+        prnt('}')
+
+    # ----------
+
+    def _convert_funcarg_to_c(self, tp, fromvar, tovar, errcode):
+        extraarg = ''
+        if isinstance(tp, model.PrimitiveType):
+            if tp.is_integer_type() and tp.name != '_Bool':
+                converter = '_cffi_to_c_int'
+                extraarg = ', %s' % tp.name
+            else:
+                converter = '(%s)_cffi_to_c_%s' % (tp.get_c_name(''),
+                                                   tp.name.replace(' ', '_'))
+            errvalue = '-1'
+        #
+        elif isinstance(tp, model.PointerType):
+            self._convert_funcarg_to_c_ptr_or_array(tp, fromvar,
+                                                    tovar, errcode)
+            return
+        #
+        elif isinstance(tp, (model.StructOrUnion, model.EnumType)):
+            # a struct (not a struct pointer) as a function argument
+            self._prnt('  if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)'
+                      % (tovar, self._gettypenum(tp), fromvar))
+            self._prnt('    %s;' % errcode)
+            return
+        #
+        elif isinstance(tp, model.FunctionPtrType):
+            converter = '(%s)_cffi_to_c_pointer' % tp.get_c_name('')
+            extraarg = ', _cffi_type(%d)' % self._gettypenum(tp)
+            errvalue = 'NULL'
+        #
+        else:
+            raise NotImplementedError(tp)
+        #
+        self._prnt('  %s = %s(%s%s);' % (tovar, converter, fromvar, extraarg))
+        self._prnt('  if (%s == (%s)%s && PyErr_Occurred())' % (
+            tovar, tp.get_c_name(''), errvalue))
+        self._prnt('    %s;' % errcode)
+
+    def _extra_local_variables(self, tp, localvars):
+        if isinstance(tp, model.PointerType):
+            localvars.add('Py_ssize_t datasize')
+
+    def _convert_funcarg_to_c_ptr_or_array(self, tp, fromvar, tovar, errcode):
+        self._prnt('  datasize = _cffi_prepare_pointer_call_argument(')
+        self._prnt('      _cffi_type(%d), %s, (char **)&%s);' % (
+            self._gettypenum(tp), fromvar, tovar))
+        self._prnt('  if (datasize != 0) {')
+        self._prnt('    if (datasize < 0)')
+        self._prnt('      %s;' % errcode)
+        self._prnt('    %s = alloca((size_t)datasize);' % (tovar,))
+        self._prnt('    memset((void *)%s, 0, (size_t)datasize);' % (tovar,))
+        self._prnt('    if (_cffi_convert_array_from_object('
+                   '(char *)%s, _cffi_type(%d), %s) < 0)' % (
+            tovar, self._gettypenum(tp), fromvar))
+        self._prnt('      %s;' % errcode)
+        self._prnt('  }')
+
+    def _convert_expr_from_c(self, tp, var, context):
+        if isinstance(tp, model.PrimitiveType):
+            if tp.is_integer_type():
+                return '_cffi_from_c_int(%s, %s)' % (var, tp.name)
+            elif tp.name != 'long double':
+                return '_cffi_from_c_%s(%s)' % (tp.name.replace(' ', '_'), var)
+            else:
+                return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % (
+                    var, self._gettypenum(tp))
+        elif isinstance(tp, (model.PointerType, model.FunctionPtrType)):
+            return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
+                var, self._gettypenum(tp))
+        elif isinstance(tp, model.ArrayType):
+            return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
+                var, self._gettypenum(model.PointerType(tp.item)))
+        elif isinstance(tp, model.StructType):
+            if tp.fldnames is None:
+                raise TypeError("'%s' is used as %s, but is opaque" % (
+                    tp._get_c_name(), context))
+            return '_cffi_from_c_struct((char *)&%s, _cffi_type(%d))' % (
+                var, self._gettypenum(tp))
+        elif isinstance(tp, model.EnumType):
+            return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % (
+                var, self._gettypenum(tp))
+        else:
+            raise NotImplementedError(tp)
+
+    # ----------
+    # function declarations
+
+    def _generate_cpy_function_decl(self, tp, name):
+        assert isinstance(tp, model.FunctionPtrType)
+        if tp.ellipsis:
+            # cannot support vararg functions better than this: check for its
+            # exact type (including the fixed arguments), and build it as a
+            # constant function pointer (no CPython wrapper)
+            self._generate_cpy_const(False, name, tp)
+            return
+        prnt = self._prnt
+        numargs = len(tp.args)
+        if numargs == 0:
+            argname = 'noarg'
+        elif numargs == 1:
+            argname = 'arg0'
+        else:
+            argname = 'args'
+        prnt('static PyObject *')
+        prnt('_cffi_f_%s(PyObject *self, PyObject *%s)' % (name, argname))
+        prnt('{')
+        #
+        context = 'argument of %s' % name
+        for i, type in enumerate(tp.args):
+            prnt('  %s;' % type.get_c_name(' x%d' % i, context))
+        #
+        localvars = set()
+        for type in tp.args:
+            self._extra_local_variables(type, localvars)
+        for decl in localvars:
+            prnt('  %s;' % (decl,))
+        #
+        if not isinstance(tp.result, model.VoidType):
+            result_code = 'result = '
+            context = 'result of %s' % name
+            prnt('  %s;' % tp.result.get_c_name(' result', context))
+        else:
+            result_code = ''
+        #
+        if len(tp.args) > 1:
+            rng = range(len(tp.args))
+            for i in rng:
+                prnt('  PyObject *arg%d;' % i)
+            prnt()
+            prnt('  if (!PyArg_ParseTuple(args, "%s:%s", %s))' % (
+                'O' * numargs, name, ', '.join(['&arg%d' % i for i in rng])))
+            prnt('    return NULL;')
+        prnt()
+        #
+        for i, type in enumerate(tp.args):
+            self._convert_funcarg_to_c(type, 'arg%d' % i, 'x%d' % i,
+                                       'return NULL')
+            prnt()
+        #
+        prnt('  Py_BEGIN_ALLOW_THREADS')
+        prnt('  _cffi_restore_errno();')
+        prnt('  { %s%s(%s); }' % (
+            result_code, name,
+            ', '.join(['x%d' % i for i in range(len(tp.args))])))
+        prnt('  _cffi_save_errno();')
+        prnt('  Py_END_ALLOW_THREADS')
+        prnt()
+        #
+        prnt('  (void)self; /* unused */')
+        if numargs == 0:
+            prnt('  (void)noarg; /* unused */')
+        if result_code:
+            prnt('  return %s;' %
+                 self._convert_expr_from_c(tp.result, 'result', 'result type'))
+        else:
+            prnt('  Py_INCREF(Py_None);')
+            prnt('  return Py_None;')
+        prnt('}')
+        prnt()
+
+    def _generate_cpy_function_global(self, tp, name):
+        if tp.ellipsis:
+            XXX
+        type_index = self._typesdict[tp.as_raw_function()]
+        numargs = len(tp.args)
+        if numargs == 0:
+            meth_kind = 'N'   # 'METH_NOARGS'
+        elif numargs == 1:
+            meth_kind = 'O'   # 'METH_O'
+        else:
+            meth_kind = 'V'   # 'METH_VARARGS'
+        self._lst.append(
+            '  { "%s", _cffi_f_%s, _CFFI_OP(_CFFI_OP_CPYTHON_BLTN_%s, %d)},'
+            % (name, name, meth_kind, type_index))
+
 
 def make_c_source(ffi, target_c_file, preamble):
-    recompiler = Recompiler(ffi)
+    module_name, ext = os.path.splitext(os.path.basename(target_c_file))
+    assert ext, "no extension!"
+    recompiler = Recompiler(ffi, module_name)
     recompiler.collect_type_table()
     with open(target_c_file, 'w') as f:
         recompiler.write_source_to_f(f, preamble)
diff --git a/new/test_recompiler.py b/new/test_recompiler.py
--- a/new/test_recompiler.py
+++ b/new/test_recompiler.py
@@ -6,7 +6,7 @@
 def check_type_table(input, expected_output):
     ffi = FFI()
     ffi.cdef(input)
-    recompiler = Recompiler(ffi)
+    recompiler = Recompiler(ffi, 'testmod')
     recompiler.collect_type_table()
     assert ''.join(map(str, recompiler.cffi_types)) == expected_output
 
@@ -38,5 +38,5 @@
 
 def test_math_sin():
     ffi = FFI()
-    ffi.cdef("double sin(double);")
+    ffi.cdef("float sin(double); double cos(double);")
     make_c_source(ffi, str(udir.join('math_sin.c')), '#include ')

From noreply at buildbot.pypy.org  Wed Apr 15 19:14:52 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 15 Apr 2015 19:14:52 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: next test
Message-ID: <20150415171452.1AD491C00FC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1715:631dcc7cd174
Date: 2015-04-15 19:01 +0200
http://bitbucket.org/cffi/cffi/changeset/631dcc7cd174/

Log:	next test

diff --git a/new/recompiler.py b/new/recompiler.py
--- a/new/recompiler.py
+++ b/new/recompiler.py
@@ -106,6 +106,10 @@
     def _emit_bytecode_PointerType(self, tp, index):
         self.cffi_types[index] = CffiOp(OP_POINTER, self._typesdict[tp.totype])
 
+    def _emit_bytecode_FunctionPtrType(self, tp, index):
+        raw = tp.as_raw_function()
+        self.cffi_types[index] = CffiOp(OP_POINTER, self._typesdict[raw])
+
     # ----------
 
     def _prnt(self, what=''):
diff --git a/new/test_recompiler.py b/new/test_recompiler.py
--- a/new/test_recompiler.py
+++ b/new/test_recompiler.py
@@ -30,11 +30,20 @@
     check_type_table("double sin(double *, double *);",
                      "(FUNCTION 4)(POINTER 4)(NOOP 1)(FUNCTION_END 0)"
                      "(PRIMITIVE 14)")
+    check_type_table("double sin(double *, double *, double);",
+                     "(FUNCTION 3)(POINTER 3)(NOOP 1)(PRIMITIVE 14)"
+                     "(FUNCTION_END 0)")
 
 def test_dont_use_noop_for_primitives():
     check_type_table("double sin(double, double);",
                      "(FUNCTION 1)(PRIMITIVE 14)(PRIMITIVE 14)(FUNCTION_END 0)")
 
+def test_funcptr_as_argument():
+    check_type_table("int sin(double(float));",
+                     "(FUNCTION 6)(PRIMITIVE 13)(FUNCTION_END 0)"
+                     "(FUNCTION 7)(POINTER 0)(FUNCTION_END 0)"
+                     "(PRIMITIVE 14)(PRIMITIVE 7)")
+
 
 def test_math_sin():
     ffi = FFI()

From noreply at buildbot.pypy.org  Wed Apr 15 19:14:53 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 15 Apr 2015 19:14:53 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Global array variables
Message-ID: <20150415171453.26D6F1C00FC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1716:4a6a3ce48e98
Date: 2015-04-15 19:15 +0200
http://bitbucket.org/cffi/cffi/changeset/4a6a3ce48e98/

Log:	Global array variables

diff --git a/new/cffi_opcode.py b/new/cffi_opcode.py
--- a/new/cffi_opcode.py
+++ b/new/cffi_opcode.py
@@ -4,11 +4,14 @@
         self.op = op
         self.arg = arg
     def as_c_expr(self):
+        if self.op is None:
+            assert isinstance(self.arg, str)
+            return '(_cffi_opcode_t)(%s)' % (self.arg,)
         classname = CLASS_NAME[self.op]
         return '_CFFI_OP(_CFFI_OP_%s, %d)' % (classname, self.arg)
     def __str__(self):
         classname = CLASS_NAME.get(self.op, self.op)
-        return '(%s %d)' % (classname, self.arg)
+        return '(%s %s)' % (classname, self.arg)
 
 OP_PRIMITIVE       = 1
 OP_POINTER         = 3
diff --git a/new/recompiler.py b/new/recompiler.py
--- a/new/recompiler.py
+++ b/new/recompiler.py
@@ -83,33 +83,6 @@
                 model.attach_exception_info(e, name)
                 raise
 
-    def _generate_cpy_function_collecttype(self, tp, name):
-        self._do_collect_type(tp.as_raw_function())
-
-    def _emit_bytecode_PrimitiveType(self, tp, index):
-        prim_index = PRIMITIVE_TO_INDEX[tp.name]
-        self.cffi_types[index] = CffiOp(OP_PRIMITIVE, prim_index)
-
-    def _emit_bytecode_RawFunctionType(self, tp, index):
-        self.cffi_types[index] = CffiOp(OP_FUNCTION, self._typesdict[tp.result])
-        index += 1
-        for tp1 in tp.args:
-            realindex = self._typesdict[tp1]
-            if index != realindex:
-                if isinstance(tp1, model.PrimitiveType):
-                    self._emit_bytecode_PrimitiveType(tp1, index)
-                else:
-                    self.cffi_types[index] = CffiOp(OP_NOOP, realindex)
-            index += 1
-        self.cffi_types[index] = CffiOp(OP_FUNCTION_END, tp.ellipsis)
-
-    def _emit_bytecode_PointerType(self, tp, index):
-        self.cffi_types[index] = CffiOp(OP_POINTER, self._typesdict[tp.totype])
-
-    def _emit_bytecode_FunctionPtrType(self, tp, index):
-        raw = tp.as_raw_function()
-        self.cffi_types[index] = CffiOp(OP_POINTER, self._typesdict[raw])
-
     # ----------
 
     def _prnt(self, what=''):
@@ -291,6 +264,9 @@
     # ----------
     # function declarations
 
+    def _generate_cpy_function_collecttype(self, tp, name):
+        self._do_collect_type(tp.as_raw_function())
+
     def _generate_cpy_function_decl(self, tp, name):
         assert isinstance(tp, model.FunctionPtrType)
         if tp.ellipsis:
@@ -379,6 +355,56 @@
             '  { "%s", _cffi_f_%s, _CFFI_OP(_CFFI_OP_CPYTHON_BLTN_%s, %d)},'
             % (name, name, meth_kind, type_index))
 
+    # ----------
+    # global variables
+
+    def _generate_cpy_variable_collecttype(self, tp, name):
+        self._do_collect_type(tp)
+
+    def _generate_cpy_variable_decl(self, tp, name):
+        pass
+
+    def _generate_cpy_variable_global(self, tp, name):
+        type_index = self._typesdict[tp]
+        self._lst.append(
+            '  { "%s", &%s, _CFFI_OP(_CFFI_OP_NOOP, %d)},'
+            % (name, name, type_index))
+
+    # ----------
+    # emitting the opcodes for individual types
+
+    def _emit_bytecode_PrimitiveType(self, tp, index):
+        prim_index = PRIMITIVE_TO_INDEX[tp.name]
+        self.cffi_types[index] = CffiOp(OP_PRIMITIVE, prim_index)
+
+    def _emit_bytecode_RawFunctionType(self, tp, index):
+        self.cffi_types[index] = CffiOp(OP_FUNCTION, self._typesdict[tp.result])
+        index += 1
+        for tp1 in tp.args:
+            realindex = self._typesdict[tp1]
+            if index != realindex:
+                if isinstance(tp1, model.PrimitiveType):
+                    self._emit_bytecode_PrimitiveType(tp1, index)
+                else:
+                    self.cffi_types[index] = CffiOp(OP_NOOP, realindex)
+            index += 1
+        self.cffi_types[index] = CffiOp(OP_FUNCTION_END, int(tp.ellipsis))
+
+    def _emit_bytecode_PointerType(self, tp, index):
+        self.cffi_types[index] = CffiOp(OP_POINTER, self._typesdict[tp.totype])
+
+    def _emit_bytecode_FunctionPtrType(self, tp, index):
+        raw = tp.as_raw_function()
+        self.cffi_types[index] = CffiOp(OP_POINTER, self._typesdict[raw])
+
+    def _emit_bytecode_ArrayType(self, tp, index):
+        item_index = self._typesdict[tp.item]
+        if tp.length is None:
+            self.cffi_types[index] = CffiOp(OP_OPEN_ARRAY, item_index)
+        else:
+            assert self.cffi_types[index + 1] == 'LEN'
+            self.cffi_types[index] = CffiOp(OP_ARRAY, item_index)
+            self.cffi_types[index + 1] = CffiOp(None, '%d' % (tp.length,))
 
 def make_c_source(ffi, target_c_file, preamble):
     module_name, ext = os.path.splitext(os.path.basename(target_c_file))
diff --git a/new/test_recompiler.py b/new/test_recompiler.py
--- a/new/test_recompiler.py
+++ b/new/test_recompiler.py
@@ -44,8 +44,17 @@
                      "(FUNCTION 7)(POINTER 0)(FUNCTION_END 0)"
                      "(PRIMITIVE 14)(PRIMITIVE 7)")
 
+def test_array():
+    check_type_table("int a[100];",
+                     "(PRIMITIVE 7)(ARRAY 0)(None 100)")
+
 
 def test_math_sin():
     ffi = FFI()
     ffi.cdef("float sin(double); double cos(double);")
     make_c_source(ffi, str(udir.join('math_sin.c')), '#include ')
+
+def test_global_var_array():
+    ffi = FFI()
+    ffi.cdef("int a[100];")
+    make_c_source(ffi, str(udir.join('global_var_array.c')), 'int a[100];')

From noreply at buildbot.pypy.org  Wed Apr 15 19:25:36 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 15 Apr 2015 19:25:36 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: A test to verify that cffi_opcopde.py
 and parse_c_type.h contain the same thing
Message-ID: <20150415172536.355311C00FC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1717:b4c788fecdc9
Date: 2015-04-15 19:26 +0200
http://bitbucket.org/cffi/cffi/changeset/b4c788fecdc9/

Log:	A test to verify that cffi_opcopde.py and parse_c_type.h contain the
	same thing

diff --git a/new/cffi_opcode.py b/new/cffi_opcode.py
--- a/new/cffi_opcode.py
+++ b/new/cffi_opcode.py
@@ -24,9 +24,10 @@
 OP_FUNCTION_END    = 17
 OP_NOOP            = 19
 OP_BITFIELD        = 21
-OP_CPYTHON_BLTN_V  = 23   # varargs
-OP_CPYTHON_BLTN_N  = 25   # noargs
-OP_CPYTHON_BLTN_O  = 27   # O  (i.e. a single arg)
+OP_INTEGER_CONST   = 23
+OP_CPYTHON_BLTN_V  = 25   # varargs
+OP_CPYTHON_BLTN_N  = 27   # noargs
+OP_CPYTHON_BLTN_O  = 29   # O  (i.e. a single arg)
 
 PRIM_VOID          = 0
 PRIM_BOOL          = 1
@@ -44,6 +45,7 @@
 PRIM_FLOAT         = 13
 PRIM_DOUBLE        = 14
 PRIM_LONGDOUBLE    = 15
+_NUM_PRIM          = 16
 
 PRIMITIVE_TO_INDEX = {
     'int':    PRIM_INT,
diff --git a/new/parse_c_type.h b/new/parse_c_type.h
--- a/new/parse_c_type.h
+++ b/new/parse_c_type.h
@@ -18,9 +18,10 @@
 #define _CFFI_OP_FUNCTION_END   17
 #define _CFFI_OP_NOOP           19
 #define _CFFI_OP_BITFIELD       21
-#define _CFFI_OP_CPYTHON_BLTN_V 23   // varargs
-#define _CFFI_OP_CPYTHON_BLTN_N 25   // noargs
-#define _CFFI_OP_CPYTHON_BLTN_O 27   // O  (i.e. a single arg)
+#define _CFFI_OP_INTEGER_CONST  23
+#define _CFFI_OP_CPYTHON_BLTN_V 25   // varargs
+#define _CFFI_OP_CPYTHON_BLTN_N 27   // noargs
+#define _CFFI_OP_CPYTHON_BLTN_O 29   // O  (i.e. a single arg)
 
 #define _CFFI_PRIM_VOID          0
 #define _CFFI_PRIM_BOOL          1
diff --git a/new/test_parse_c_type.py b/new/test_parse_c_type.py
--- a/new/test_parse_c_type.py
+++ b/new/test_parse_c_type.py
@@ -242,3 +242,9 @@
         assert parse("%s" % (identifier_names[i])) == ['->', Typename(i)]
         assert parse("%s*" % (identifier_names[i])) == [Typename(i),
                                                         '->', Pointer(0)]
+
+def test_cffi_opcode_sync():
+    import cffi_opcode
+    for name in dir(lib):
+        if name.startswith('_CFFI_'):
+            assert getattr(cffi_opcode, name[6:]) == getattr(lib, name)

From noreply at buildbot.pypy.org  Wed Apr 15 19:48:46 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Wed, 15 Apr 2015 19:48:46 +0200 (CEST)
Subject: [pypy-commit] pypy callfamily: replace bk.enter()/bk.leave() pairs
 with a context manager
Message-ID: <20150415174846.8B8531C0362@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: callfamily
Changeset: r76798:1ebe0debf9ff
Date: 2015-04-15 18:48 +0100
http://bitbucket.org/pypy/pypy/changeset/1ebe0debf9ff/

Log:	replace bk.enter()/bk.leave() pairs with a context manager

diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -407,8 +407,7 @@
             i = 0
             while i < len(block.operations):
                 op = block.operations[i]
-                self.bookkeeper.enter((graph, block, i))
-                try:
+                with self.bookkeeper.at_position((graph, block, i)):
                     new_ops = op.transform(self)
                     if new_ops is not None:
                         block.operations[i:i+1] = new_ops
@@ -417,8 +416,6 @@
                         new_ops[-1].result = op.result
                         op = new_ops[0]
                     self.consider_op(op)
-                finally:
-                    self.bookkeeper.leave()
                 i += 1
 
         except BlockedInference as e:
diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -5,6 +5,7 @@
 from __future__ import absolute_import
 
 import sys, types, inspect, weakref
+from contextlib import contextmanager
 
 from rpython.flowspace.model import Constant
 from rpython.annotator.model import (SomeOrderedDict,
@@ -87,11 +88,22 @@
         del TLS.bookkeeper
         del self.position_key
 
+    @contextmanager
+    def at_position(self, pos):
+        """A context manager calling `self.enter()` and `self.leave()`"""
+        if hasattr(self, 'position_key') and pos is None:
+            yield
+            return
+        self.enter(pos)
+        try:
+            yield
+        finally:
+            self.leave()
+
     def compute_at_fixpoint(self):
         # getbookkeeper() needs to work during this function, so provide
         # one with a dummy position
-        self.enter(None)
-        try:
+        with self.at_position(None):
             for call_op in self.annotator.call_sites():
                 self.consider_call_site(call_op)
 
@@ -99,8 +111,6 @@
                 args = simple_args(args_s)
                 pbc.consider_call_site(args, s_ImpossibleValue, None)
             self.emulated_pbc_calls = {}
-        finally:
-            self.leave()
 
     def check_no_flags_on_instances(self):
         # sanity check: no flags attached to heap stored instances
@@ -480,7 +490,6 @@
         """Analyse a call to a SomePBC() with the given args (list of
         annotations).
         """
-        descs = list(pbc.descriptions)
         if emulated is None:
             whence = self.position_key
             # fish the existing annotation for the result variable,
@@ -505,10 +514,7 @@
         return s_result
 
     def emulate_pbc_call(self, unique_key, pbc, args_s, replace=[], callback=None):
-        emulate_enter = not hasattr(self, 'position_key')
-        if emulate_enter:
-            self.enter(None)
-        try:
+        with self.at_position(None):
             emulated_pbc_calls = self.emulated_pbc_calls
             prev = [unique_key]
             prev.extend(replace)
@@ -523,9 +529,6 @@
             else:
                 emulated = callback
             return self.pbc_call(pbc, args, emulated=emulated)
-        finally:
-            if emulate_enter:
-                self.leave()
 
     def _find_current_op(self, opname=None, arity=None, pos=None, s_type=None):
         """ Find operation that is currently being annotated. Do some
diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py
--- a/rpython/annotator/description.py
+++ b/rpython/annotator/description.py
@@ -361,11 +361,8 @@
     @staticmethod
     def variant_for_call_site(bookkeeper, family, descs, args, op):
         shape = rawshape(args)
-        bookkeeper.enter(None)
-        try:
+        with bookkeeper.at_position(None):
             row = FunctionDesc.row_to_consider(descs, args, op)
-        finally:
-            bookkeeper.leave()
         index = family.calltable_lookup_row(shape, row)
         return shape, index
 

From noreply at buildbot.pypy.org  Wed Apr 15 19:58:27 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 15 Apr 2015 19:58:27 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Tweaks to use a model that is closer
 to the existing one of cffi than an
Message-ID: <20150415175827.33BD81C0362@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1718:eec843d871af
Date: 2015-04-15 19:59 +0200
http://bitbucket.org/cffi/cffi/changeset/eec843d871af/

Log:	Tweaks to use a model that is closer to the existing one of cffi
	than an ideal model would be

diff --git a/new/cffi_opcode.py b/new/cffi_opcode.py
--- a/new/cffi_opcode.py
+++ b/new/cffi_opcode.py
@@ -19,15 +19,14 @@
 OP_OPEN_ARRAY      = 7
 OP_STRUCT_UNION    = 9
 OP_ENUM            = 11
-OP_TYPENAME        = 13
-OP_FUNCTION        = 15
-OP_FUNCTION_END    = 17
-OP_NOOP            = 19
-OP_BITFIELD        = 21
-OP_INTEGER_CONST   = 23
-OP_CPYTHON_BLTN_V  = 25   # varargs
-OP_CPYTHON_BLTN_N  = 27   # noargs
-OP_CPYTHON_BLTN_O  = 29   # O  (i.e. a single arg)
+OP_FUNCTION        = 13
+OP_FUNCTION_END    = 15
+OP_NOOP            = 17
+OP_BITFIELD        = 19
+OP_INTEGER_CONST   = 21
+OP_CPYTHON_BLTN_V  = 23   # varargs
+OP_CPYTHON_BLTN_N  = 25   # noargs
+OP_CPYTHON_BLTN_O  = 27   # O  (i.e. a single arg)
 
 PRIM_VOID          = 0
 PRIM_BOOL          = 1
@@ -45,12 +44,54 @@
 PRIM_FLOAT         = 13
 PRIM_DOUBLE        = 14
 PRIM_LONGDOUBLE    = 15
-_NUM_PRIM          = 16
+
+PRIM_WCHAR         = 16
+PRIM_INT8          = 17
+PRIM_UINT8         = 18
+PRIM_INT16         = 19
+PRIM_UINT16        = 20
+PRIM_INT32         = 21
+PRIM_UINT32        = 22
+PRIM_INT64         = 23
+PRIM_UINT64        = 24
+PRIM_INTPTR        = 25
+PRIM_UINTPTR       = 26
+PRIM_PTRDIFF       = 27
+PRIM_SIZE          = 28
+PRIM_SSIZE         = 29
+
+_NUM_PRIM          = 30
 
 PRIMITIVE_TO_INDEX = {
-    'int':    PRIM_INT,
-    'float':  PRIM_FLOAT,
-    'double': PRIM_DOUBLE,
+    'char':               PRIM_CHAR,
+    'short':              PRIM_SHORT,
+    'int':                PRIM_INT,
+    'long':               PRIM_LONG,
+    'long long':          PRIM_LONGLONG,
+    'signed char':        PRIM_SCHAR,
+    'unsigned char':      PRIM_UCHAR,
+    'unsigned short':     PRIM_USHORT,
+    'unsigned int':       PRIM_UINT,
+    'unsigned long':      PRIM_ULONG,
+    'unsigned long long': PRIM_ULONGLONG,
+    'float':              PRIM_FLOAT,
+    'double':             PRIM_DOUBLE,
+    'long double':        PRIM_LONGDOUBLE,
+    '_Bool':              PRIM_BOOL,
+    'wchar_t':            PRIM_WCHAR,
+    'int8_t':             PRIM_INT8,
+    'uint8_t':            PRIM_UINT8,
+    'int16_t':            PRIM_INT16,
+    'uint16_t':           PRIM_UINT16,
+    'int32_t':            PRIM_INT32,
+    'uint32_t':           PRIM_UINT32,
+    'int64_t':            PRIM_INT64,
+    'uint64_t':           PRIM_UINT64,
+    'intptr_t':           PRIM_INTPTR,
+    'uintptr_t':          PRIM_UINTPTR,
+    'ptrdiff_t':          PRIM_PTRDIFF,
+    'size_t':             PRIM_SIZE,
+    'ssize_t':            PRIM_SSIZE,
     }
 
 CLASS_NAME = {}
diff --git a/new/parse_c_type.c b/new/parse_c_type.c
--- a/new/parse_c_type.c
+++ b/new/parse_c_type.c
@@ -506,7 +506,8 @@
             if (n < 0)
                 return parse_error(tok, "undefined type name");
 
-            t1 = _CFFI_OP(_CFFI_OP_TYPENAME, n);
+            t1 = _CFFI_OP(_CFFI_OP_NOOP,
+                          tok->info->ctx->typenames[n].type_index);
             break;
         }
         case TOK_STRUCT:
diff --git a/new/parse_c_type.h b/new/parse_c_type.h
--- a/new/parse_c_type.h
+++ b/new/parse_c_type.h
@@ -13,15 +13,14 @@
 #define _CFFI_OP_OPEN_ARRAY      7
 #define _CFFI_OP_STRUCT_UNION    9
 #define _CFFI_OP_ENUM           11
-#define _CFFI_OP_TYPENAME       13
-#define _CFFI_OP_FUNCTION       15
-#define _CFFI_OP_FUNCTION_END   17
-#define _CFFI_OP_NOOP           19
-#define _CFFI_OP_BITFIELD       21
-#define _CFFI_OP_INTEGER_CONST  23
-#define _CFFI_OP_CPYTHON_BLTN_V 25   // varargs
-#define _CFFI_OP_CPYTHON_BLTN_N 27   // noargs
-#define _CFFI_OP_CPYTHON_BLTN_O 29   // O  (i.e. a single arg)
+#define _CFFI_OP_FUNCTION       13
+#define _CFFI_OP_FUNCTION_END   15
+#define _CFFI_OP_NOOP           17
+#define _CFFI_OP_BITFIELD       19
+#define _CFFI_OP_INTEGER_CONST  21
+#define _CFFI_OP_CPYTHON_BLTN_V 23   // varargs
+#define _CFFI_OP_CPYTHON_BLTN_N 25   // noargs
+#define _CFFI_OP_CPYTHON_BLTN_O 27   // O  (i.e. a single arg)
 
 #define _CFFI_PRIM_VOID          0
 #define _CFFI_PRIM_BOOL          1
@@ -39,7 +38,23 @@
 #define _CFFI_PRIM_FLOAT        13
 #define _CFFI_PRIM_DOUBLE       14
 #define _CFFI_PRIM_LONGDOUBLE   15
-#define _CFFI__NUM_PRIM         16
+
+#define _CFFI_PRIM_WCHAR        16
+#define _CFFI_PRIM_INT8         17
+#define _CFFI_PRIM_UINT8        18
+#define _CFFI_PRIM_INT16        19
+#define _CFFI_PRIM_UINT16       20
+#define _CFFI_PRIM_INT32        21
+#define _CFFI_PRIM_UINT32       22
+#define _CFFI_PRIM_INT64        23
+#define _CFFI_PRIM_UINT64       24
+#define _CFFI_PRIM_INTPTR       25
+#define _CFFI_PRIM_UINTPTR      26
+#define _CFFI_PRIM_PTRDIFF      27
+#define _CFFI_PRIM_SIZE         28
+#define _CFFI_PRIM_SSIZE        29
+
+#define _CFFI__NUM_PRIM         30
 
 
 struct _cffi_global_s {
@@ -79,7 +94,8 @@
 
 struct _cffi_typename_s {
     const char *name;
-    _cffi_opcode_t type_op;   /* 0 if opaque or prebuilt */
+    int type_index;   /* if opaque, points to a possibly artificial
+                         OP_STRUCT which is itself opaque */
 };
 
 struct _cffi_type_context_s {
diff --git a/new/recompiler.py b/new/recompiler.py
--- a/new/recompiler.py
+++ b/new/recompiler.py
@@ -11,7 +11,7 @@
 
     def collect_type_table(self):
         self._typesdict = {}
-        self._generate('collecttype')
+        self._generate("collecttype")
         #
         all_decls = sorted(self._typesdict, key=str)
         #
@@ -262,6 +262,12 @@
             raise NotImplementedError(tp)
 
     # ----------
+    # typedefs
+
+    def _generate_cpy_typedef_collecttype(self, tp, name):
+        self._do_collect_type(tp)
+
+    # ----------
     # function declarations
 
     def _generate_cpy_function_collecttype(self, tp, name):
diff --git a/new/test_parse_c_type.py b/new/test_parse_c_type.py
--- a/new/test_parse_c_type.py
+++ b/new/test_parse_c_type.py
@@ -37,6 +37,7 @@
 ctx_identifiers = ffi.new("struct _cffi_typename_s[]", len(identifier_names))
 for _i in range(len(identifier_names)):
     ctx_identifiers[_i].name = c_identifier_names[_i]
+    ctx_identifiers[_i].type_index = 100 + _i
 ctx.typenames = ctx_identifiers
 ctx.num_typenames = len(identifier_names)
 
@@ -92,7 +93,6 @@
 Func = make_getter('FUNCTION')
 FuncEnd = make_getter('FUNCTION_END')
 Struct = make_getter('STRUCT_UNION')
-Typename = make_getter('TYPENAME')
 
 
 def test_simple():
@@ -239,12 +239,14 @@
 
 def test_identifier():
     for i in range(len(identifier_names)):
-        assert parse("%s" % (identifier_names[i])) == ['->', Typename(i)]
-        assert parse("%s*" % (identifier_names[i])) == [Typename(i),
+        assert parse("%s" % (identifier_names[i])) == ['->', NoOp(100 + i)]
+        assert parse("%s*" % (identifier_names[i])) == [NoOp(100 + i),
                                                         '->', Pointer(0)]
 
 def test_cffi_opcode_sync():
-    import cffi_opcode
+    import cffi_opcode, cffi1.model
     for name in dir(lib):
         if name.startswith('_CFFI_'):
             assert getattr(cffi_opcode, name[6:]) == getattr(lib, name)
+    assert sorted(cffi_opcode.PRIMITIVE_TO_INDEX.keys()) == (
+        sorted(cffi1.model.PrimitiveType.ALL_PRIMITIVE_TYPES.keys()))
diff --git a/new/test_recompiler.py b/new/test_recompiler.py
--- a/new/test_recompiler.py
+++ b/new/test_recompiler.py
@@ -44,10 +44,22 @@
                      "(FUNCTION 7)(POINTER 0)(FUNCTION_END 0)"
                      "(PRIMITIVE 14)(PRIMITIVE 7)")
 
+def test_variadic_function():
+    check_type_table("int sin(int, ...);",
+                     "(FUNCTION 1)(PRIMITIVE 7)(FUNCTION_END 1)")
+
 def test_array():
     check_type_table("int a[100];",
                      "(PRIMITIVE 7)(ARRAY 0)(None 100)")
 
+def test_typedef():
+    check_type_table("typedef int foo_t;",
+                     "(PRIMITIVE 7)")
+
+def test_prebuilt_type():
+    check_type_table("int32_t f(void);",
+                     "(FUNCTION 2)(FUNCTION_END 0)(PRIMITIVE 21)")
+
 
 def test_math_sin():
     ffi = FFI()

From noreply at buildbot.pypy.org  Wed Apr 15 20:09:43 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 15 Apr 2015 20:09:43 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Generate struct _cffi_typename_s
Message-ID: <20150415180943.23F3E1C00FC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1719:31ea57b2d53d
Date: 2015-04-15 20:10 +0200
http://bitbucket.org/cffi/cffi/changeset/31ea57b2d53d/

Log:	Generate struct _cffi_typename_s

diff --git a/new/ffi_obj.c b/new/ffi_obj.c
--- a/new/ffi_obj.c
+++ b/new/ffi_obj.c
@@ -64,7 +64,7 @@
         const void *mem[] = {ffi->info.ctx->types,
                              ffi->info.ctx->globals,
                              ffi->info.ctx->constants,
-                             ffi->info.ctx->structs_unions,
+                             ffi->info.ctx->struct_unions,
                              ffi->info.ctx->fields,
                              ffi->info.ctx->enums,
                              ffi->info.ctx->typenames,
diff --git a/new/parse_c_type.c b/new/parse_c_type.c
--- a/new/parse_c_type.c
+++ b/new/parse_c_type.c
@@ -371,7 +371,7 @@
   }
 
 MAKE_SEARCH_FUNC(globals)
-MAKE_SEARCH_FUNC(structs_unions)
+MAKE_SEARCH_FUNC(struct_unions)
 MAKE_SEARCH_FUNC(typenames)
 
 #undef MAKE_SEARCH_FUNC
@@ -518,10 +518,10 @@
             if (tok->kind != TOK_IDENTIFIER)
                 return parse_error(tok, "struct or union name expected");
 
-            int n = search_in_structs_unions(tok->info->ctx, tok->p, tok->size);
+            int n = search_in_struct_unions(tok->info->ctx, tok->p, tok->size);
             if (n < 0)
                 return parse_error(tok, "undefined struct/union name");
-            if (((tok->info->ctx->structs_unions[n].flags & CT_UNION) != 0)
+            if (((tok->info->ctx->struct_unions[n].flags & CT_UNION) != 0)
                 ^ (kind == TOK_UNION))
                 return parse_error(tok, "wrong kind of tag: struct vs union");
 
diff --git a/new/parse_c_type.h b/new/parse_c_type.h
--- a/new/parse_c_type.h
+++ b/new/parse_c_type.h
@@ -102,13 +102,13 @@
     _cffi_opcode_t *types;
     const struct _cffi_global_s *globals;
     const struct _cffi_constant_s *constants;
-    const struct _cffi_struct_union_s *structs_unions;
+    const struct _cffi_struct_union_s *struct_unions;
     const struct _cffi_field_s *fields;
     const struct _cffi_enum_s *enums;
     const struct _cffi_typename_s *typenames;
     int num_globals;
     int num_constants;
-    int num_structs_unions;
+    int num_struct_unions;
     int num_enums;
     int num_typenames;
 };
diff --git a/new/recompiler.py b/new/recompiler.py
--- a/new/recompiler.py
+++ b/new/recompiler.py
@@ -116,54 +116,42 @@
         # ffi._parser._declarations.  This generates all the functions.
         self._generate("decl")
         #
-        # the declaration of '_cffi_globals'
-        self._lst = []
-        self._generate("global")
-        num_globals = len(self._lst)
-        if num_globals > 0:
-            self._lst.sort()  # sort by name, which is at the start of each line
-            prnt('static const struct _cffi_global_s _cffi_globals[] = {')
-            for line in self._lst:
-                prnt(line)
-            prnt('};')
-            prnt()
+        # the declaration of '_cffi_globals' and '_cffi_typenames'
+        nums = {}
+        self._lsts = {"global": [], "typename": []}
+        self._generate("ctx")
+        for step_name in ["global", "typename"]:
+            lst = self._lsts[step_name]
+            nums[step_name] = len(lst)
+            if nums[step_name] > 0:
+                lst.sort()  # sort by name, which is at the start of each line
+                prnt('static const struct _cffi_%s_s _cffi_%s[] = {' % (
+                    step_name, step_name))
+                for line in lst:
+                    prnt(line)
+                prnt('};')
+                prnt()
         #
         # XXX
-        num_constants = 0
-        num_structs_unions = 0
-        num_enums = 0
-        num_typenames = 0
+        nums['constant'] = 0
+        nums['struct_union'] = 0
+        nums['enum'] = 0
         #
         # the declaration of '_cffi_type_context'
         prnt('static const struct _cffi_type_context_s _cffi_type_context = {')
         prnt('  _cffi_types,')
-        if num_globals > 0:
-            prnt('  _cffi_globals,')
-        else:
-            prnt('  NULL,  /* no globals */')
-        if num_constants > 0:
-            prnt('  _cffi_constants,')
-        else:
-            prnt('  NULL,  /* no constants */')
-        if num_structs_unions > 0:
-            prnt('  _cffi_structs_unions,')
-            prnt('  _cffi_fields,')
-        else:
-            prnt('  NULL,  /* no structs */')
-            prnt('  NULL,  /* no fields */')
-        if num_enums > 0:
-            prnt('  _cffi_enums,')
-        else:
-            prnt('  NULL,  /* no enums */')
-        if num_typenames > 0:
-            prnt('  _cffi_typenames,')
-        else:
-            prnt('  NULL,  /* no typenames */')
-        prnt('  %d,  /* num_globals */' % num_globals)
-        prnt('  %d,  /* num_constants */' % num_constants)
-        prnt('  %d,  /* num_structs_unions */' % num_structs_unions)
-        prnt('  %d,  /* num_enums */' % num_enums)
-        prnt('  %d,  /* num_typenames */' % num_typenames)
+        ALL_STEPS = ["global", "constant", "struct_union", "enum", "typename"]
+        for step_name in ALL_STEPS:
+            if nums[step_name] > 0:
+                prnt('  _cffi_%ss,' % step_name)
+                if step_name == 'struct_union':
+                    prnt('  _cffi_fields,')
+            else:
+                prnt('  NULL,  /* no %ss */' % step_name)
+                if step_name == 'struct_union':
+                    prnt('  NULL,  /* no fields */')
+        for step_name in ALL_STEPS:
+            prnt('  %d,  /* num_%ss */' % (nums[step_name], step_name))
         prnt('};')
         prnt()
         #
@@ -267,6 +255,14 @@
     def _generate_cpy_typedef_collecttype(self, tp, name):
         self._do_collect_type(tp)
 
+    def _generate_cpy_typedef_decl(self, tp, name):
+        pass
+
+    def _generate_cpy_typedef_ctx(self, tp, name):
+        type_index = self._typesdict[tp]
+        self._lsts["typename"].append(
+            '  { "%s", %d },' % (name, type_index))
+
     # ----------
     # function declarations
 
@@ -346,7 +342,7 @@
         prnt('}')
         prnt()
 
-    def _generate_cpy_function_global(self, tp, name):
+    def _generate_cpy_function_ctx(self, tp, name):
         if tp.ellipsis:
             XXX
         type_index = self._typesdict[tp.as_raw_function()]
@@ -357,7 +353,7 @@
             meth_kind = 'O'   # 'METH_O'
         else:
             meth_kind = 'V'   # 'METH_VARARGS'
-        self._lst.append(
+        self._lsts["global"].append(
             '  { "%s", _cffi_f_%s, _CFFI_OP(_CFFI_OP_CPYTHON_BLTN_%s, %d)},'
             % (name, name, meth_kind, type_index))
 
@@ -370,9 +366,9 @@
     def _generate_cpy_variable_decl(self, tp, name):
         pass
 
-    def _generate_cpy_variable_global(self, tp, name):
+    def _generate_cpy_variable_ctx(self, tp, name):
         type_index = self._typesdict[tp]
-        self._lst.append(
+        self._lsts["global"].append(
             '  { "%s", &%s, _CFFI_OP(_CFFI_OP_NOOP, %d)},'
             % (name, name, type_index))
 
diff --git a/new/test_parse_c_type.py b/new/test_parse_c_type.py
--- a/new/test_parse_c_type.py
+++ b/new/test_parse_c_type.py
@@ -30,8 +30,8 @@
 for _i in range(len(struct_names)):
     ctx_structs[_i].name = c_struct_names[_i]
 ctx_structs[3].flags = lib.CT_UNION
-ctx.structs_unions = ctx_structs
-ctx.num_structs_unions = len(struct_names)
+ctx.struct_unions = ctx_structs
+ctx.num_struct_unions = len(struct_names)
 
 c_identifier_names = [ffi.new("char[]", _n) for _n in identifier_names]
 ctx_identifiers = ffi.new("struct _cffi_typename_s[]", len(identifier_names))
diff --git a/new/test_recompiler.py b/new/test_recompiler.py
--- a/new/test_recompiler.py
+++ b/new/test_recompiler.py
@@ -70,3 +70,8 @@
     ffi = FFI()
     ffi.cdef("int a[100];")
     make_c_source(ffi, str(udir.join('global_var_array.c')), 'int a[100];')
+
+def test_typedef():
+    ffi = FFI()
+    ffi.cdef("typedef int **foo_t;")
+    make_c_source(ffi, str(udir.join('typedef.c')), 'typedef int **foo_t;')

From noreply at buildbot.pypy.org  Wed Apr 15 20:36:57 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 15 Apr 2015 20:36:57 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Introduce the standard typedefs as
 "primitives" too, as cffi always did
Message-ID: <20150415183657.06D6B1C02A3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1720:c2879301170e
Date: 2015-04-15 20:37 +0200
http://bitbucket.org/cffi/cffi/changeset/c2879301170e/

Log:	Introduce the standard typedefs as "primitives" too, as cffi always
	did

diff --git a/new/parse_c_type.c b/new/parse_c_type.c
--- a/new/parse_c_type.c
+++ b/new/parse_c_type.c
@@ -377,6 +377,70 @@
 #undef MAKE_SEARCH_FUNC
 
 
+int search_standard_typename(const char *p, size_t size)
+{
+    if (size < 6 || p[size-2] != '_' || p[size-1] != 't')
+        return -1;
+
+    switch (p[4]) {
+
+    case '1':
+        if (size == 8 && !memcmp(p, "uint16", 6)) return _CFFI_PRIM_UINT16;
+        break;
+
+    case '2':
+        if (size == 7 && !memcmp(p, "int32", 5)) return _CFFI_PRIM_INT32;
+        break;
+
+    case '3':
+        if (size == 8 && !memcmp(p, "uint32", 6)) return _CFFI_PRIM_UINT32;
+        break;
+
+    case '4':
+        if (size == 7 && !memcmp(p, "int64", 5)) return _CFFI_PRIM_INT64;
+        break;
+
+    case '6':
+        if (size == 8 && !memcmp(p, "uint64", 6)) return _CFFI_PRIM_UINT64;
+        if (size == 7 && !memcmp(p, "int16", 5)) return _CFFI_PRIM_INT16;
+        break;
+
+    case '8':
+        if (size == 7 && !memcmp(p, "uint8", 5)) return _CFFI_PRIM_UINT8;
+        break;
+
+    case 'e':
+        if (size == 7 && !memcmp(p, "ssize", 5)) return _CFFI_PRIM_SSIZE;
+        break;
+
+    case 'i':
+        if (size == 9 && !memcmp(p, "ptrdiff", 7)) return _CFFI_PRIM_PTRDIFF;
+        break;
+
+    case 'p':
+        if (size == 9 && !memcmp(p, "uintptr", 7)) return _CFFI_PRIM_UINTPTR;
+        break;
+
+    case 'r':
+        if (size == 7 && !memcmp(p, "wchar", 5)) return _CFFI_PRIM_WCHAR;
+        break;
+
+    case 't':
+        if (size == 8 && !memcmp(p, "intptr", 6)) return _CFFI_PRIM_INTPTR;
+        break;
+
+    case '_':
+        if (size == 6 && !memcmp(p, "size", 4)) return _CFFI_PRIM_SIZE;
+        if (size == 6 && !memcmp(p, "int8", 4)) return _CFFI_PRIM_INT8;
+        break;
+
+    default:
+        break;
+    }
+    return -1;
+}
+
+
 static int parse_complete(token_t *tok)
 {
  qualifiers:
@@ -503,12 +567,17 @@
         case TOK_IDENTIFIER:
         {
             int n = search_in_typenames(tok->info->ctx, tok->p, tok->size);
-            if (n < 0)
-                return parse_error(tok, "undefined type name");
-
-            t1 = _CFFI_OP(_CFFI_OP_NOOP,
-                          tok->info->ctx->typenames[n].type_index);
-            break;
+            if (n >= 0) {
+                t1 = _CFFI_OP(_CFFI_OP_NOOP,
+                              tok->info->ctx->typenames[n].type_index);
+                break;
+            }
+            n = search_standard_typename(tok->p, tok->size);
+            if (n >= 0) {
+                t1 = _CFFI_OP(_CFFI_OP_PRIMITIVE, n);
+                break;
+            }
+            return parse_error(tok, "undefined type name");
         }
         case TOK_STRUCT:
         case TOK_UNION:
diff --git a/new/realize_c_type.c b/new/realize_c_type.c
--- a/new/realize_c_type.c
+++ b/new/realize_c_type.c
@@ -4,23 +4,50 @@
 
 PyObject *build_primitive_type(int num)
 {
+    /* XXX too many translations between here and new_primitive_type() */
+    static const char *primitive_name[] = {
+        NULL,
+        "_Bool",
+        "char",
+        "signed char",
+        "unsigned char",
+        "short",
+        "unsigned short",
+        "int",
+        "unsigned int",
+        "long",
+        "unsigned long",
+        "long long",
+        "unsigned long long",
+        "float",
+        "double",
+        "long double",
+        "wchar_t",
+        "int8_t",
+        "uint8_t",
+        "int16_t",
+        "uint16_t",
+        "int32_t",
+        "uint32_t",
+        "int64_t",
+        "uint64_t",
+        "intptr_t",
+        "uintptr_t",
+        "ptrdiff_t",
+        "size_t",
+        "ssize_t",
+    };
     PyObject *x;
 
-    switch (num) {
-
-    case _CFFI_PRIM_VOID:
+    if (num == _CFFI_PRIM_VOID) {
         x = new_void_type();
-        break;
-
-    case _CFFI_PRIM_INT:
-        x = new_primitive_type("int");
-        break;
-
-    case _CFFI_PRIM_LONG:
-        x = new_primitive_type("long");
-        break;
-
-    default:
+    }
+    else if (0 <= num &&
+             num < sizeof(primitive_name) / sizeof(*primitive_name) &&
+             primitive_name[num] != NULL) {
+        x = new_primitive_type(primitive_name[num]);
+    }
+    else {
         PyErr_Format(PyExc_NotImplementedError, "prim=%d", num);
         return NULL;
     }
diff --git a/new/test_realize_c_type.py b/new/test_realize_c_type.py
--- a/new/test_realize_c_type.py
+++ b/new/test_realize_c_type.py
@@ -41,3 +41,8 @@
     check("int(*)(int(int))", "int(*)(int(*)(int))")
     check("int(*)(long[])", "int(*)(long *)")
     check("int(*)(long[5])", "int(*)(long *)")
+
+def test_all_primitives():
+    import cffi_opcode
+    for name in cffi_opcode.PRIMITIVE_TO_INDEX:
+        check(name, name)

From noreply at buildbot.pypy.org  Thu Apr 16 01:00:30 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Thu, 16 Apr 2015 01:00:30 +0200 (CEST)
Subject: [pypy-commit] pypy object-dtype2: kill stupid hack
Message-ID: <20150415230030.3AA9E1C0498@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: object-dtype2
Changeset: r76799:275a1cc62537
Date: 2015-04-15 23:54 +0100
http://bitbucket.org/pypy/pypy/changeset/275a1cc62537/

Log:	kill stupid hack

diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -112,13 +112,12 @@
     return dispatcher
 
 class BaseType(object):
-    _immutable_fields_ = ['native', 'space', 'T']
+    _immutable_fields_ = ['native', 'space']
 
     def __init__(self, space, native=True):
         assert isinstance(space, ObjSpace)
         self.native = native
         self.space = space
-        self.T = self.T
 
     def __repr__(self):
         return self.__class__.__name__
@@ -1746,7 +1745,7 @@
 
     def _obool(self, v):
         if self.space.is_true(v):
-            return True 
+            return True
         return False
 
     @simple_binary_op
@@ -1837,14 +1836,14 @@
 
 def add_attributeerr_op(cls, op):
     def func(self, *args):
-        raise oefmt(self.space.w_AttributeError, 
+        raise oefmt(self.space.w_AttributeError,
             "%s", op)
     func.__name__ = 'object_' + op
     setattr(cls, op, func)
-        
+
 def add_unsupported_op(cls, op):
     def func(self, *args):
-        raise oefmt(self.space.w_TypeError, 
+        raise oefmt(self.space.w_TypeError,
             "ufunc '%s' not supported for input types", op)
     func.__name__ = 'object_' + op
     setattr(cls, op, func)

From noreply at buildbot.pypy.org  Thu Apr 16 01:00:31 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Thu, 16 Apr 2015 01:00:31 +0200 (CEST)
Subject: [pypy-commit] pypy object-dtype2: Add ObjectType.byteswap() (fixes
 a translation failure)
Message-ID: <20150415230031.624D11C0498@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: object-dtype2
Changeset: r76800:9c75809161df
Date: 2015-04-15 23:56 +0100
http://bitbucket.org/pypy/pypy/changeset/9c75809161df/

Log:	Add ObjectType.byteswap() (fixes a translation failure)

diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1655,6 +1655,9 @@
     def read(self, arr, i, offset, dtype=None):
         return self.box(self._read(arr.storage, i, offset))
 
+    def byteswap(self, w_v):
+        return w_v
+
     @jit.dont_look_inside
     def _write(self, storage, i, offset, w_obj, gcstruct):
         # no GC anywhere in this function!

From noreply at buildbot.pypy.org  Thu Apr 16 09:36:45 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 16 Apr 2015 09:36:45 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: bump version number
Message-ID: <20150416073645.991761C1027@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1721:b890af05b4c2
Date: 2015-04-16 08:33 +0200
http://bitbucket.org/cffi/cffi/changeset/b890af05b4c2/

Log:	bump version number

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -5813,7 +5813,7 @@
     if (v == NULL || PyModule_AddObject(m, "_C_API", v) < 0)
         INITERROR;
 
-    v = PyText_FromString("0.9.2");
+    v = PyText_FromString("1.0.0");
     if (v == NULL || PyModule_AddObject(m, "__version__", v) < 0)
         INITERROR;
 
diff --git a/cffi/__init__.py b/cffi/__init__.py
--- a/cffi/__init__.py
+++ b/cffi/__init__.py
@@ -1,13 +1,10 @@
 __all__ = ['FFI', 'VerificationError', 'VerificationMissing', 'CDefError',
            'FFIError']
 
-from .api import FFI, CDefError, FFIError
+from .api import FFI, CDefError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "0.9.2"
-__version_info__ = (0, 9, 2)
+FFIError = FFI.error    # backward compatibility
 
-# The verifier module file names are based on the CRC32 of a string that
-# contains the following version number.  It may be older than __version__
-# if nothing is clearly incompatible.
-__version_verifier_modules__ = "0.8.6"
+__version__ = "1.0.0"
+__version_info__ = (1, 0, 0)
diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -16,9 +16,6 @@
     basestring = str
 
 
-class FFIError(Exception):
-    pass
-
 class CDefError(Exception):
     def __str__(self):
         try:

From noreply at buildbot.pypy.org  Thu Apr 16 09:36:46 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 16 Apr 2015 09:36:46 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Reimplement verify() as a hack that
 updates the old ffi object with
Message-ID: <20150416073646.BAE371C1027@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1722:3758e99173fd
Date: 2015-04-16 09:17 +0200
http://bitbucket.org/cffi/cffi/changeset/3758e99173fd/

Log:	Reimplement verify() as a hack that updates the old ffi object with
	the ffi object from the extension module.

diff --git a/new/cffi1_module.c b/new/cffi1_module.c
--- a/new/cffi1_module.c
+++ b/new/cffi1_module.c
@@ -49,7 +49,7 @@
     if (m == NULL)
         return -1;
 
-    FFIObject *ffi = ffi_internal_new(NULL, ctx);
+    FFIObject *ffi = ffi_internal_new(&FFI_Type, ctx, 1);
     Py_XINCREF(ffi);    /* make the ffi object really immortal */
     if (ffi == NULL || PyModule_AddObject(m, "ffi", (PyObject *)ffi) < 0)
         return -1;
diff --git a/new/ffi_obj.c b/new/ffi_obj.c
--- a/new/ffi_obj.c
+++ b/new/ffi_obj.c
@@ -23,19 +23,21 @@
     PyObject *types_dict;
     PyObject *gc_wrefs;
     struct _cffi_parse_info_s info;
+    int ctx_is_static;
     _cffi_opcode_t internal_output[FFI_COMPLEXITY_OUTPUT];
 };
 
 static FFIObject *ffi_internal_new(PyTypeObject *ffitype,
-                                   const struct _cffi_type_context_s *ctx)
+                                   const struct _cffi_type_context_s *ctx,
+                                   int ctx_is_static)
 {
     PyObject *dict = PyDict_New();
     if (dict == NULL)
         return NULL;
 
     FFIObject *ffi;
-    if (ffitype == NULL) {
-        ffi = (FFIObject *)PyObject_New(FFIObject, &FFI_Type);
+    if (ctx_is_static) {
+        ffi = (FFIObject *)PyObject_New(FFIObject, ffitype);
         /* we don't call PyObject_GC_Track() here: from _cffi_init_module()
            it is not needed, because in this case the ffi object is immortal */
     }
@@ -51,6 +53,7 @@
     ffi->info.ctx = ctx;
     ffi->info.output = ffi->internal_output;
     ffi->info.output_size = FFI_COMPLEXITY_OUTPUT;
+    ffi->ctx_is_static = ctx_is_static;
     return ffi;
 }
 
@@ -60,7 +63,7 @@
     Py_DECREF(ffi->types_dict);
     Py_XDECREF(ffi->gc_wrefs);
 
-    {
+    if (!ffi->ctx_is_static) {
         const void *mem[] = {ffi->info.ctx->types,
                              ffi->info.ctx->globals,
                              ffi->info.ctx->constants,
@@ -98,7 +101,7 @@
     }
     memset(ctx, 0, sizeof(struct _cffi_type_context_s));
 
-    result = (PyObject *)ffi_internal_new(type, ctx);
+    result = (PyObject *)ffi_internal_new(type, ctx, 0);
     if (result == NULL) {
         PyMem_Free(ctx);
         return NULL;
@@ -516,7 +519,40 @@
 }
 #endif
 
+static PyObject *ffi__verified(FFIObject *self, PyObject *args)
+{
+    FFIObject *srcffi;
+
+    if (!PyArg_ParseTuple(args, "O!:_verified", &FFI_Type, &srcffi))
+        return NULL;
+
+    if (!srcffi->ctx_is_static)
+        goto invalid;
+
+    if (self->ctx_is_static)
+        goto invalid;
+
+    size_t i;
+    const char *p = (const char *)self->info.ctx;
+    for (i = 0; i < sizeof(struct _cffi_type_context_s); i++) {
+        if (*p++ != '\0')
+            goto invalid;
+    }
+
+    PyMem_Free((void *)self->info.ctx);
+    self->ctx_is_static = 1;
+    self->info.ctx = srcffi->info.ctx;
+
+    Py_INCREF(Py_None);
+    return Py_None;
+
+ invalid:
+    PyErr_SetString(PyExc_ValueError, "XXX invalid source or destination");
+    return NULL;
+}
+
 static PyMethodDef ffi_methods[] = {
+    {"_verified",     (PyCFunction)ffi__verified, METH_VARARGS},
 #if 0
     {"addressof",     (PyCFunction)ffi_addressof, METH_VARARGS},
     {"cast",          (PyCFunction)ffi_cast,      METH_VARARGS},
diff --git a/new/recompiler.py b/new/recompiler.py
--- a/new/recompiler.py
+++ b/new/recompiler.py
@@ -125,7 +125,7 @@
             nums[step_name] = len(lst)
             if nums[step_name] > 0:
                 lst.sort()  # sort by name, which is at the start of each line
-                prnt('static const struct _cffi_%s_s _cffi_%s[] = {' % (
+                prnt('static const struct _cffi_%s_s _cffi_%ss[] = {' % (
                     step_name, step_name))
                 for line in lst:
                     prnt(line)
@@ -408,10 +408,32 @@
             self.cffi_types[index] = CffiOp(OP_ARRAY, item_index)
             self.cffi_types[index + 1] = CffiOp(None, '%d' % (tp.length,))
 
-def make_c_source(ffi, target_c_file, preamble):
-    module_name, ext = os.path.splitext(os.path.basename(target_c_file))
-    assert ext, "no extension!"
+def make_c_source(ffi, module_name, preamble, target_c_file):
     recompiler = Recompiler(ffi, module_name)
     recompiler.collect_type_table()
     with open(target_c_file, 'w') as f:
         recompiler.write_source_to_f(f, preamble)
+
+def _get_extension(module_name, c_file, kwds):
+    source_name = ffiplatform.maybe_relative_path(c_file)
+    include_dirs = kwds.setdefault('include_dirs', [])
+    include_dirs.insert(0, '.')   # XXX
+    return ffiplatform.get_extension(source_name, module_name, **kwds)
+
+def recompile(ffi, module_name, preamble, tmpdir=None, **kwds):
+    if tmpdir is None:
+        tmpdir = 'build'
+        if not os.path.isdir(tmpdir):
+            os.mkdir(tmpdir)
+    c_file = os.path.join(tmpdir, module_name + '.c')
+    ext = _get_extension(module_name, c_file, kwds)
+    make_c_source(ffi, module_name, preamble, c_file)
+    outputfilename = ffiplatform.compile(tmpdir, ext)
+    return outputfilename
+
+def verify(ffi, module_name, preamble, *args, **kwds):
+    import imp
+    outputfilename = recompile(ffi, module_name, preamble, *args, **kwds)
+    module = imp.load_dynamic(module_name, outputfilename)
+    ffi._verified(module.ffi)
+    return module.lib
diff --git a/new/test_recompiler.py b/new/test_recompiler.py
--- a/new/test_recompiler.py
+++ b/new/test_recompiler.py
@@ -1,6 +1,5 @@
-from recompiler import Recompiler, make_c_source
+from recompiler import Recompiler, verify
 from cffi1 import FFI
-from udir import udir
 
 
 def check_type_table(input, expected_output):
@@ -62,16 +61,21 @@
 
 
 def test_math_sin():
+    import math
     ffi = FFI()
     ffi.cdef("float sin(double); double cos(double);")
-    make_c_source(ffi, str(udir.join('math_sin.c')), '#include ')
+    lib = verify(ffi, 'test_math_sin', '#include ')
+    assert lib.cos(1.43) == math.cos(1.43)
 
 def test_global_var_array():
     ffi = FFI()
     ffi.cdef("int a[100];")
-    make_c_source(ffi, str(udir.join('global_var_array.c')), 'int a[100];')
+    lib = verify(ffi, 'test_global_var_array', 'int a[100] = { 9999 };')
+    #lib.a[42] = 123456
+    #assert lib.a[42] == 123456
+    #assert lib.a[0] == 9999
 
 def test_typedef():
     ffi = FFI()
     ffi.cdef("typedef int **foo_t;")
-    make_c_source(ffi, str(udir.join('typedef.c')), 'typedef int **foo_t;')
+    lib = verify(ffi, 'test_typedef', 'typedef int **foo_t;')

From noreply at buildbot.pypy.org  Thu Apr 16 09:36:47 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 16 Apr 2015 09:36:47 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Take cglob.c from zeffir,
	and fix a few bugs
Message-ID: <20150416073647.D2D251C1027@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1723:e3ed1d0463a1
Date: 2015-04-16 09:37 +0200
http://bitbucket.org/cffi/cffi/changeset/e3ed1d0463a1/

Log:	Take cglob.c from zeffir, and fix a few bugs

diff --git a/new/cffi1_module.c b/new/cffi1_module.c
--- a/new/cffi1_module.c
+++ b/new/cffi1_module.c
@@ -10,6 +10,7 @@
 static PyTypeObject Lib_Type;   /* forward */
 
 #include "ffi_obj.c"
+#include "cglob.c"
 #include "lib_obj.c"
 
 
diff --git a/new/cglob.c b/new/cglob.c
new file mode 100644
--- /dev/null
+++ b/new/cglob.c
@@ -0,0 +1,68 @@
+
+typedef struct {
+    PyObject_HEAD
+
+    CTypeDescrObject *gs_type;
+    char             *gs_data;
+
+} GlobSupportObject;
+
+static void glob_support_dealloc(GlobSupportObject *gs)
+{
+    Py_DECREF(gs->gs_type);
+    PyObject_Del(gs);
+}
+
+static PyTypeObject GlobSupport_Type = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "cffi.GlobSupport",
+    sizeof(GlobSupportObject),
+    0,
+    (destructor)glob_support_dealloc,           /* tp_dealloc */
+    0,                                          /* tp_print */
+    0,                                          /* tp_getattr */
+    0,                                          /* tp_setattr */
+    0,                                          /* tp_compare */
+    0,                                          /* tp_repr */
+    0,                                          /* tp_as_number */
+    0,                                          /* tp_as_sequence */
+    0,                                          /* tp_as_mapping */
+    0,                                          /* tp_hash */
+    0,                                          /* tp_call */
+    0,                                          /* tp_str */
+    PyObject_GenericGetAttr,                    /* tp_getattro */
+    0,                                          /* tp_setattro */
+    0,                                          /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT,                         /* tp_flags */
+};
+
+#define GlobSupport_Check(ob)  (Py_TYPE(ob) == &GlobSupport_Type)
+
+static PyObject *make_global_var(CTypeDescrObject *type, char *addr)
+{
+    GlobSupportObject *gs = PyObject_New(GlobSupportObject, &GlobSupport_Type);
+    if (gs == NULL)
+        return NULL;
+
+    Py_INCREF(type);
+    gs->gs_type = type;
+    gs->gs_data = addr;
+    return (PyObject *)gs;
+}
+
+static PyObject *read_global_var(GlobSupportObject *gs)
+{
+    return convert_to_object(gs->gs_data, gs->gs_type);
+}
+
+#if 0
+static int write_global_var(GlobSupportObject *gs, PyObject *obj)
+{
+    return convert_from_object(gs->gs_data, gs->gs_type, obj);
+}
+
+static PyObject *addressof_global_var(GlobSupportObject *gs)
+{
+    return new_simple_cdata(gs->gs_data, gs->gs_type);
+}
+#endif
diff --git a/new/ffi_obj.c b/new/ffi_obj.c
--- a/new/ffi_obj.c
+++ b/new/ffi_obj.c
@@ -144,23 +144,24 @@
                          input_text, spaces);
             return NULL;
         }
-        PyObject *ct = realize_c_type(ffi->info.ctx,
-                                      ffi->info.output, index);
+        CTypeDescrObject *ct = realize_c_type(ffi->info.ctx,
+                                              ffi->info.output, index);
         if (ct == NULL)
             return NULL;
 
-        char *normalized_text = ((CTypeDescrObject *)ct)->ct_name;
+        char *normalized_text = ct->ct_name;
         x = PyDict_GetItemString(ffi->types_dict, normalized_text);
         if (x == NULL) {
-            PyDict_SetItemString(ffi->types_dict, normalized_text, ct);
+            PyDict_SetItemString(ffi->types_dict, normalized_text,
+                                 (PyObject *)ct);
         }
         else {
             Py_INCREF(x);
             Py_DECREF(ct);
-            ct = x;
+            ct = (CTypeDescrObject *)x;
         }
-        PyDict_SetItem(ffi->types_dict, arg, ct);
-        return (CTypeDescrObject *)ct;
+        PyDict_SetItem(ffi->types_dict, arg, (PyObject *)ct);
+        return ct;
     }
     else if ((accept & ACCEPT_CTYPE) && CTypeDescr_Check(arg)) {
         return (CTypeDescrObject *)arg;
diff --git a/new/lib_obj.c b/new/lib_obj.c
--- a/new/lib_obj.c
+++ b/new/lib_obj.c
@@ -95,6 +95,7 @@
 
     const struct _cffi_global_s *g = &lib->l_ctx->globals[index];
     PyObject *x;
+    CTypeDescrObject *ct;
 
     switch (_CFFI_GETOP(g->type_op)) {
 
@@ -110,6 +111,17 @@
         x = lib_build_cpython_func(lib, g, s, METH_O);
         break;
 
+    case _CFFI_OP_NOOP:
+        /* this is used for global variables, of the exact type specified
+           here */
+        ct = realize_c_type(lib->l_ctx, lib->l_ctx->types,
+                            _CFFI_GETARG(g->type_op));
+        if (ct == NULL)
+            return NULL;
+        x = make_global_var(ct, g->address);
+        Py_DECREF(ct);
+        break;
+
     default:
         PyErr_SetString(PyExc_NotImplementedError, "in lib_build_attr");
         return NULL;
@@ -130,9 +142,9 @@
     if (x == NULL)
         x = lib_build_and_cache_attr(lib, name);
 
-    //if (ZefGlobSupport_Check(x)) {
-    //    return read_global_var((ZefGlobSupportObject *)x);
-    //}
+    if (GlobSupport_Check(x)) {
+        return read_global_var((GlobSupportObject *)x);
+    }
     return x;
 }
 
diff --git a/new/realize_c_type.c b/new/realize_c_type.c
--- a/new/realize_c_type.c
+++ b/new/realize_c_type.c
@@ -66,13 +66,13 @@
    the intermediate types back in the opcodes[].  Returns a new
    reference.
 */
-static PyObject *
+static CTypeDescrObject *
 realize_c_type(const struct _cffi_type_context_s *ctx,
                _cffi_opcode_t opcodes[], int index)
 {
     PyObject *x = _realize_c_type_or_func(ctx, opcodes, index);
     if (x == NULL || CTypeDescr_Check(x)) {
-        return x;
+        return (CTypeDescrObject *)x;
     }
     else {
         PyObject *y;
@@ -129,10 +129,10 @@
         break;
 
     case _CFFI_OP_ARRAY:
-        length = (Py_ssize_t)opcodes[_CFFI_GETARG(op) + 1];
+        length = (Py_ssize_t)opcodes[index + 1];
         /* fall-through */
     case _CFFI_OP_OPEN_ARRAY:
-        y = realize_c_type(ctx, opcodes, _CFFI_GETARG(op));
+        y = (PyObject *)realize_c_type(ctx, opcodes, _CFFI_GETARG(op));
         if (y == NULL)
             return NULL;
         z = new_pointer_type((CTypeDescrObject *)y);
@@ -148,7 +148,7 @@
         PyObject *fargs;
         int i, base_index, num_args;
 
-        y = realize_c_type(ctx, opcodes, _CFFI_GETARG(op));
+        y = (PyObject *)realize_c_type(ctx, opcodes, _CFFI_GETARG(op));
         if (y == NULL)
             return NULL;
 
@@ -165,7 +165,7 @@
         }
 
         for (i = 0; i < num_args; i++) {
-            z = realize_c_type(ctx, opcodes, base_index + i);
+            z = (PyObject *)realize_c_type(ctx, opcodes, base_index + i);
             if (z == NULL) {
                 Py_DECREF(fargs);
                 Py_DECREF(y);
diff --git a/new/test_realize_c_type.py b/new/test_realize_c_type.py
--- a/new/test_realize_c_type.py
+++ b/new/test_realize_c_type.py
@@ -28,7 +28,7 @@
     check("int(*)", "int *")
 
 def test_array():
-    check("int[5]")
+    check("int[6]")
 
 def test_funcptr():
     check("int(*)(long)")
diff --git a/new/test_recompiler.py b/new/test_recompiler.py
--- a/new/test_recompiler.py
+++ b/new/test_recompiler.py
@@ -71,9 +71,9 @@
     ffi = FFI()
     ffi.cdef("int a[100];")
     lib = verify(ffi, 'test_global_var_array', 'int a[100] = { 9999 };')
-    #lib.a[42] = 123456
-    #assert lib.a[42] == 123456
-    #assert lib.a[0] == 9999
+    lib.a[42] = 123456
+    assert lib.a[42] == 123456
+    assert lib.a[0] == 9999
 
 def test_typedef():
     ffi = FFI()

From noreply at buildbot.pypy.org  Thu Apr 16 09:53:58 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 16 Apr 2015 09:53:58 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: setting globals; listing globals
Message-ID: <20150416075358.373AA1C1234@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1724:f897a98f6448
Date: 2015-04-16 09:54 +0200
http://bitbucket.org/cffi/cffi/changeset/f897a98f6448/

Log:	setting globals; listing globals

diff --git a/new/cglob.c b/new/cglob.c
--- a/new/cglob.c
+++ b/new/cglob.c
@@ -55,12 +55,12 @@
     return convert_to_object(gs->gs_data, gs->gs_type);
 }
 
-#if 0
 static int write_global_var(GlobSupportObject *gs, PyObject *obj)
 {
     return convert_from_object(gs->gs_data, gs->gs_type, obj);
 }
 
+#if 0
 static PyObject *addressof_global_var(GlobSupportObject *gs)
 {
     return new_simple_cdata(gs->gs_data, gs->gs_type);
diff --git a/new/lib_obj.c b/new/lib_obj.c
--- a/new/lib_obj.c
+++ b/new/lib_obj.c
@@ -139,8 +139,11 @@
 static PyObject *lib_getattr(LibObject *lib, PyObject *name)
 {
     PyObject *x = PyDict_GetItem(lib->l_dict, name);
-    if (x == NULL)
+    if (x == NULL) {
         x = lib_build_and_cache_attr(lib, name);
+        if (x == NULL)
+            return NULL;
+    }
 
     if (GlobSupport_Check(x)) {
         return read_global_var((GlobSupportObject *)x);
@@ -148,21 +151,22 @@
     return x;
 }
 
-#if 0
-static int lib_setattr(ZefLibObject *lib, PyObject *name, PyObject *val)
+static int lib_setattr(LibObject *lib, PyObject *name, PyObject *val)
 {
-    PyObject *x = lib_findattr(lib, name, PyExc_AttributeError);
-    if (x == NULL)
-        return -1;
+    PyObject *x = PyDict_GetItem(lib->l_dict, name);
+    if (x == NULL) {
+        x = lib_build_and_cache_attr(lib, name);
+        if (x == NULL)
+            return -1;
+    }
 
     if (val == NULL) {
-        PyErr_SetString(PyExc_AttributeError,
-                        "cannot delete attributes from Lib object");
+        PyErr_SetString(PyExc_AttributeError, "C attribute cannot be deleted");
         return -1;
     }
 
-    if (ZefGlobSupport_Check(x)) {
-        return write_global_var((ZefGlobSupportObject *)x, val);
+    if (GlobSupport_Check(x)) {
+        return write_global_var((GlobSupportObject *)x, val);
     }
 
     PyErr_Format(PyExc_AttributeError,
@@ -171,16 +175,31 @@
     return -1;
 }
 
-static PyObject *lib_dir(PyObject *lib, PyObject *noarg)
+static PyObject *lib_dir(LibObject *lib, PyObject *noarg)
 {
-    return PyDict_Keys(((ZefLibObject *)lib)->l_dict);
+    const struct _cffi_global_s *g = lib->l_ctx->globals;
+    int total = lib->l_ctx->num_globals;
+
+    PyObject *lst = PyList_New(total);
+    if (lst == NULL)
+        return NULL;
+
+    int i;
+    for (i = 0; i < total; i++) {
+        PyObject *s = PyString_FromString(g[i].name);
+        if (s == NULL) {
+            Py_DECREF(lst);
+            return NULL;
+        }
+        PyList_SET_ITEM(lst, i, s);
+    }
+    return lst;
 }
 
 static PyMethodDef lib_methods[] = {
-    {"__dir__",   lib_dir,  METH_NOARGS},
+    {"__dir__",   (PyCFunction)lib_dir,  METH_NOARGS},
     {NULL,        NULL}           /* sentinel */
 };
-#endif
 
 static PyTypeObject Lib_Type = {
     PyVarObject_HEAD_INIT(NULL, 0)
@@ -200,11 +219,7 @@
     0,                                          /* tp_call */
     0,                                          /* tp_str */
     (getattrofunc)lib_getattr,                  /* tp_getattro */
-#if 0 // XXX
     (setattrofunc)lib_setattr,                  /* tp_setattro */
-#else
-    0,
-#endif
     0,                                          /* tp_as_buffer */
     Py_TPFLAGS_DEFAULT,                         /* tp_flags */
     0,                                          /* tp_doc */
@@ -214,11 +229,7 @@
     0,                                          /* tp_weaklistoffset */
     0,                                          /* tp_iter */
     0,                                          /* tp_iternext */
-#if 0 // XXX
     lib_methods,                                /* tp_methods */
-#else
-    0,
-#endif
     0,                                          /* tp_members */
     0,                                          /* tp_getset */
     0,                                          /* tp_base */
diff --git a/new/recompiler.py b/new/recompiler.py
--- a/new/recompiler.py
+++ b/new/recompiler.py
@@ -1,4 +1,4 @@
-import os
+import os, sys
 from cffi1 import ffiplatform, model
 from cffi_opcode import *
 
@@ -433,6 +433,8 @@
 
 def verify(ffi, module_name, preamble, *args, **kwds):
     import imp
+    assert module_name not in sys.modules, "module name conflict: %r" % (
+        module_name,)
     outputfilename = recompile(ffi, module_name, preamble, *args, **kwds)
     module = imp.load_dynamic(module_name, outputfilename)
     ffi._verified(module.ffi)
diff --git a/new/test_recompiler.py b/new/test_recompiler.py
--- a/new/test_recompiler.py
+++ b/new/test_recompiler.py
@@ -1,3 +1,4 @@
+import py
 from recompiler import Recompiler, verify
 from cffi1 import FFI
 
@@ -79,3 +80,27 @@
     ffi = FFI()
     ffi.cdef("typedef int **foo_t;")
     lib = verify(ffi, 'test_typedef', 'typedef int **foo_t;')
+
+def test_global_var_int():
+    ffi = FFI()
+    ffi.cdef("int a, b;")
+    lib = verify(ffi, 'test_global_var_int', 'int a = 999, b;')
+    assert lib.a == 999
+    lib.a -= 1001
+    assert lib.a == -2
+    lib.a = -2147483648
+    assert lib.a == -2147483648
+    py.test.raises(OverflowError, "lib.a = 2147483648")
+    py.test.raises(OverflowError, "lib.a = -2147483649")
+    lib.b = 525      # try with the first access being in setattr, too
+    assert lib.b == 525
+
+def test_dir():
+    ffi = FFI()
+    ffi.cdef("int ff(int); int aa;")
+    lib = verify(ffi, 'test_dir', """
+        int aa;
+        int ff(int x) { return x+aa; }
+    """)
+    lib.aa = 5
+    assert dir(lib) == ['aa', 'ff']

From noreply at buildbot.pypy.org  Thu Apr 16 10:04:01 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 16 Apr 2015 10:04:01 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: thanks fijal
Message-ID: <20150416080401.8ED2F1C12E8@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1725:9b29a6a05459
Date: 2015-04-16 10:04 +0200
http://bitbucket.org/cffi/cffi/changeset/9b29a6a05459/

Log:	thanks fijal

diff --git a/new/test_recompiler.py b/new/test_recompiler.py
--- a/new/test_recompiler.py
+++ b/new/test_recompiler.py
@@ -76,10 +76,10 @@
     assert lib.a[42] == 123456
     assert lib.a[0] == 9999
 
-def test_typedef():
+def test_verify_typedef():
     ffi = FFI()
     ffi.cdef("typedef int **foo_t;")
-    lib = verify(ffi, 'test_typedef', 'typedef int **foo_t;')
+    lib = verify(ffi, 'test_verify_typedef', 'typedef int **foo_t;')
 
 def test_global_var_int():
     ffi = FFI()

From noreply at buildbot.pypy.org  Thu Apr 16 10:05:32 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 16 Apr 2015 10:05:32 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Improve test
Message-ID: <20150416080532.BCF601C12E8@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1726:807538ed086d
Date: 2015-04-16 10:06 +0200
http://bitbucket.org/cffi/cffi/changeset/807538ed086d/

Log:	Improve test

diff --git a/new/test_recompiler.py b/new/test_recompiler.py
--- a/new/test_recompiler.py
+++ b/new/test_recompiler.py
@@ -83,8 +83,8 @@
 
 def test_global_var_int():
     ffi = FFI()
-    ffi.cdef("int a, b;")
-    lib = verify(ffi, 'test_global_var_int', 'int a = 999, b;')
+    ffi.cdef("int a, b, c;")
+    lib = verify(ffi, 'test_global_var_int', 'int a = 999, b, c;')
     assert lib.a == 999
     lib.a -= 1001
     assert lib.a == -2
@@ -94,6 +94,9 @@
     py.test.raises(OverflowError, "lib.a = -2147483649")
     lib.b = 525      # try with the first access being in setattr, too
     assert lib.b == 525
+    py.test.raises(AttributeError, "del lib.a")
+    py.test.raises(AttributeError, "del lib.c")
+    py.test.raises(AttributeError, "del lib.foobarbaz")
 
 def test_dir():
     ffi = FFI()

From noreply at buildbot.pypy.org  Thu Apr 16 10:17:50 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 16 Apr 2015 10:17:50 +0200 (CEST)
Subject: [pypy-commit] pypy default: backout vmprof backout
Message-ID: <20150416081750.CB5541C071E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r76801:cd3483547e8f
Date: 2015-04-16 10:13 +0200
http://bitbucket.org/pypy/pypy/changeset/cd3483547e8f/

Log:	backout vmprof backout

diff too long, truncating to 2000 out of 2680 lines

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -420,3 +420,10 @@
 the terms of the GPL license version 2 or any later version.  Thus the
 gdbm module, provided in the file lib_pypy/gdbm.py, is redistributed
 under the terms of the GPL license as well.
+
+License for 'pypy/module/_vmprof/src'
+--------------------------------------
+
+The code is based on gperftools. You may see a copy of the License for it at
+
+    https://code.google.com/p/gperftools/source/browse/COPYING
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -38,6 +38,9 @@
     "_csv", "cppyy", "_pypyjson"
 ])
 
+if sys.platform.startswith('linux') and sys.maxint > 2147483647:
+    working_modules.add('_vmprof')
+
 translation_modules = default_modules.copy()
 translation_modules.update([
     "fcntl", "time", "select", "signal", "_rawffi", "zlib", "struct", "_md5",
@@ -99,6 +102,7 @@
     "_hashlib"  : ["pypy.module._ssl.interp_ssl"],
     "_minimal_curses": ["pypy.module._minimal_curses.fficurses"],
     "_continuation": ["rpython.rlib.rstacklet"],
+    "_vmprof" : ["pypy.module._vmprof.interp_vmprof"],
     }
 
 def get_module_validator(modname):
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -9,6 +9,7 @@
 from rpython.rlib.signature import signature
 from rpython.rlib.rarithmetic import r_uint, SHRT_MIN, SHRT_MAX, \
     INT_MIN, INT_MAX, UINT_MAX, USHRT_MAX
+from rpython.rlib.rweaklist import RWeakListMixin
 
 from pypy.interpreter.executioncontext import (ExecutionContext, ActionFlag,
     UserDelAction)
@@ -366,6 +367,10 @@
 
 # ____________________________________________________________
 
+class CodeObjWeakList(RWeakListMixin):
+    def __init__(self):
+        self.initialize()
+
 class ObjSpace(object):
     """Base class for the interpreter-level implementations of object spaces.
     http://pypy.readthedocs.org/en/latest/objspace.html"""
@@ -389,6 +394,7 @@
         self.check_signal_action = None   # changed by the signal module
         self.user_del_action = UserDelAction(self)
         self._code_of_sys_exc_info = None
+        self.all_code_objs = CodeObjWeakList()
 
         # can be overridden to a subclass
         self.initialize()
@@ -666,6 +672,16 @@
             assert ec is not None
             return ec
 
+    def register_code_object(self, pycode):
+        callback = self.getexecutioncontext().register_code_callback
+        if callback is not None:
+            callback(self, pycode)
+        self.all_code_objs.add_handle(pycode)
+
+    def set_code_callback(self, callback):
+        ec = self.getexecutioncontext()
+        ec.register_code_callback = callback
+        
     def _freeze_(self):
         return True
 
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -33,6 +33,11 @@
         self.profilefunc = None
         self.w_profilefuncarg = None
         self.thread_disappeared = False   # might be set to True after os.fork()
+        self.register_code_callback = None
+        if sys.maxint == 2147483647:
+            self._code_unique_id = 0 # XXX this is wrong, it won't work on 32bit
+        else:
+            self._code_unique_id = 0x7000000000000000
 
     @staticmethod
     def _mark_thread_disappeared(space):
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -14,9 +14,10 @@
     CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS, CO_NESTED,
     CO_GENERATOR, CO_KILL_DOCSTRING, CO_YIELD_INSIDE_TRY)
 from pypy.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT
-from rpython.rlib.rarithmetic import intmask
+from rpython.rlib.rarithmetic import intmask, r_longlong
 from rpython.rlib.objectmodel import compute_hash
 from rpython.rlib import jit
+from rpython.rlib.debug import debug_start, debug_stop, debug_print
 
 
 class BytecodeCorruption(Exception):
@@ -54,8 +55,9 @@
     "CPython-style code objects."
     _immutable_ = True
     _immutable_fields_ = ["co_consts_w[*]", "co_names_w[*]", "co_varnames[*]",
-                          "co_freevars[*]", "co_cellvars[*]", "_args_as_cellvars[*]"]
-
+                          "co_freevars[*]", "co_cellvars[*]",
+                          "_args_as_cellvars[*]"]
+    
     def __init__(self, space,  argcount, nlocals, stacksize, flags,
                      code, consts, names, varnames, filename,
                      name, firstlineno, lnotab, freevars, cellvars,
@@ -83,6 +85,7 @@
         self.magic = magic
         self._signature = cpython_code_signature(self)
         self._initialize()
+        space.register_code_object(self)
 
     def _initialize(self):
         if self.co_cellvars:
@@ -124,6 +127,15 @@
             from pypy.objspace.std.mapdict import init_mapdict_cache
             init_mapdict_cache(self)
 
+        ec = self.space.getexecutioncontext()
+        self._unique_id = ec._code_unique_id
+        ec._code_unique_id += 2 # so we have one bit that we can mark stuff
+        # with
+
+    def _get_full_name(self):
+        return "py:%s:%d:%s" % (self.co_name, self.co_firstlineno,
+                                self.co_filename)
+
     def _cleanup_(self):
         if (self.magic == cpython_magic and
             '__pypy__' not in sys.builtin_module_names):
diff --git a/pypy/module/_vmprof/__init__.py b/pypy/module/_vmprof/__init__.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/__init__.py
@@ -0,0 +1,18 @@
+from pypy.interpreter.mixedmodule import MixedModule
+
+class Module(MixedModule):
+    """
+    Write me :)
+    """
+    appleveldefs = {
+    }
+
+    interpleveldefs = {
+        'enable': 'interp_vmprof.enable',
+        'disable': 'interp_vmprof.disable',
+    }
+
+    def setup_after_space_initialization(self):
+        # force the __extend__ hacks to occur early
+        from pypy.module._vmprof.interp_vmprof import VMProf
+        self.vmprof = VMProf()
diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -0,0 +1,223 @@
+import py, os, sys
+from rpython.rtyper.lltypesystem import lltype, rffi, llmemory
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+from rpython.rtyper.annlowlevel import cast_instance_to_gcref, cast_base_ptr_to_instance
+from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib import jit, rposix, entrypoint
+from rpython.rtyper.tool import rffi_platform as platform
+from rpython.rlib.rstring import StringBuilder
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.error import oefmt, wrap_oserror, OperationError
+from pypy.interpreter.gateway import unwrap_spec
+from pypy.interpreter.pyframe import PyFrame
+
+ROOT = py.path.local(__file__).join('..')
+SRC = ROOT.join('src')
+
+# by default, we statically link vmprof.c into pypy; however, if you set
+# DYNAMIC_VMPROF to True, it will be dynamically linked to the libvmprof.so
+# which is expected to be inside pypy/module/_vmprof/src: this is very useful
+# during development. Note that you have to manually build libvmprof by
+# running make inside the src dir
+DYNAMIC_VMPROF = False
+
+eci_kwds = dict(
+    include_dirs = [SRC],
+    includes = ['vmprof.h', 'trampoline.h'],
+    separate_module_files = [SRC.join('trampoline.asmgcc.s')],
+    libraries = ['unwind'],
+    
+    post_include_bits=["""
+        void* pypy_vmprof_get_virtual_ip(void*);
+        void pypy_vmprof_init(void);
+    """],
+    
+    separate_module_sources=["""
+        void pypy_vmprof_init(void) {
+            vmprof_set_mainloop(pypy_execute_frame_trampoline, 0,
+                                pypy_vmprof_get_virtual_ip);
+        }
+    """],
+    )
+
+
+if DYNAMIC_VMPROF:
+    eci_kwds['libraries'] += ['vmprof']
+    eci_kwds['link_extra'] = ['-Wl,-rpath,%s' % SRC, '-L%s' % SRC]
+else:
+    eci_kwds['separate_module_files'] += [SRC.join('vmprof.c')]
+
+eci = ExternalCompilationInfo(**eci_kwds)
+
+check_eci = eci.merge(ExternalCompilationInfo(separate_module_files=[
+    SRC.join('fake_pypy_api.c')]))
+
+platform.verify_eci(check_eci)
+
+pypy_execute_frame_trampoline = rffi.llexternal(
+    "pypy_execute_frame_trampoline",
+    [llmemory.GCREF, llmemory.GCREF, llmemory.GCREF],
+    llmemory.GCREF,
+    compilation_info=eci,
+    _nowrapper=True, sandboxsafe=True,
+    random_effects_on_gcobjs=True)
+
+pypy_vmprof_init = rffi.llexternal("pypy_vmprof_init", [], lltype.Void,
+                                   compilation_info=eci)
+vmprof_enable = rffi.llexternal("vmprof_enable",
+                                [rffi.INT, rffi.LONG, rffi.INT,
+                                 rffi.CCHARP, rffi.INT],
+                                rffi.INT, compilation_info=eci,
+                                save_err=rffi.RFFI_SAVE_ERRNO)
+vmprof_disable = rffi.llexternal("vmprof_disable", [], rffi.INT,
+                                 compilation_info=eci,
+                                save_err=rffi.RFFI_SAVE_ERRNO)
+
+vmprof_register_virtual_function = rffi.llexternal(
+    "vmprof_register_virtual_function",
+    [rffi.CCHARP, rffi.VOIDP, rffi.VOIDP], lltype.Void,
+    compilation_info=eci, _nowrapper=True)
+
+original_execute_frame = PyFrame.execute_frame.im_func
+original_execute_frame.c_name = 'pypy_pyframe_execute_frame'
+original_execute_frame._dont_inline_ = True
+
+class __extend__(PyFrame):
+    def execute_frame(frame, w_inputvalue=None, operr=None):
+        # go through the asm trampoline ONLY if we are translated but not being JITted.
+        #
+        # If we are not translated, we obviously don't want to go through the
+        # trampoline because there is no C function it can call.
+        #
+        # If we are being JITted, we want to skip the trampoline, else the JIT
+        # cannot see throug it
+        if we_are_translated() and not jit.we_are_jitted():
+            # if we are translated, call the trampoline
+            gc_frame = cast_instance_to_gcref(frame)
+            gc_inputvalue = cast_instance_to_gcref(w_inputvalue)
+            gc_operr = cast_instance_to_gcref(operr)
+            gc_result = pypy_execute_frame_trampoline(gc_frame, gc_inputvalue, gc_operr)
+            return cast_base_ptr_to_instance(W_Root, gc_result)
+        else:
+            return original_execute_frame(frame, w_inputvalue, operr)
+
+
+ at entrypoint.entrypoint_lowlevel('main', [llmemory.GCREF],
+                                'pypy_vmprof_get_virtual_ip', True)
+def get_virtual_ip(gc_frame):
+    frame = cast_base_ptr_to_instance(PyFrame, gc_frame)
+    if jit._get_virtualizable_token(frame):
+        return rffi.cast(rffi.VOIDP, 0)
+    virtual_ip = do_get_virtual_ip(frame)
+    return rffi.cast(rffi.VOIDP, virtual_ip)
+
+def do_get_virtual_ip(frame):
+    return frame.pycode._unique_id
+
+def write_long_to_string_builder(l, b):
+    if sys.maxint == 2147483647:
+        b.append(chr(l & 0xff))
+        b.append(chr((l >> 8) & 0xff))
+        b.append(chr((l >> 16) & 0xff))
+        b.append(chr((l >> 24) & 0xff))
+    else:
+        b.append(chr(l & 0xff))
+        b.append(chr((l >> 8) & 0xff))
+        b.append(chr((l >> 16) & 0xff))
+        b.append(chr((l >> 24) & 0xff))
+        b.append(chr((l >> 32) & 0xff))
+        b.append(chr((l >> 40) & 0xff))
+        b.append(chr((l >> 48) & 0xff))
+        b.append(chr((l >> 56) & 0xff))
+
+class VMProf(object):
+    def __init__(self):
+        self.is_enabled = False
+        self.ever_enabled = False
+        self.mapping_so_far = [] # stored mapping in between runs
+        self.fileno = -1
+
+    def enable(self, space, fileno, period):
+        if self.is_enabled:
+            raise oefmt(space.w_ValueError, "_vmprof already enabled")
+        self.fileno = fileno
+        self.is_enabled = True
+        self.write_header(fileno, period)
+        if not self.ever_enabled:
+            if we_are_translated():
+                pypy_vmprof_init()
+            self.ever_enabled = True
+        for weakcode in space.all_code_objs.get_all_handles():
+            code = weakcode()
+            if code:
+                self.register_code(space, code)
+        space.set_code_callback(vmprof_register_code)
+        if we_are_translated():
+            # does not work untranslated
+            res = vmprof_enable(fileno, period, 0,
+                                lltype.nullptr(rffi.CCHARP.TO), 0)
+        else:
+            res = 0
+        if res == -1:
+            raise wrap_oserror(space, OSError(rposix.get_saved_errno(),
+                                              "_vmprof.enable"))
+
+    def write_header(self, fileno, period):
+        if period == -1:
+            period_usec = 1000000 / 100 #  100hz
+        else:
+            period_usec = period
+        b = StringBuilder()
+        write_long_to_string_builder(0, b)
+        write_long_to_string_builder(3, b)
+        write_long_to_string_builder(0, b)
+        write_long_to_string_builder(period_usec, b)
+        write_long_to_string_builder(0, b)
+        os.write(fileno, b.build())
+
+    def register_code(self, space, code):
+        if self.fileno == -1:
+            raise OperationError(space.w_RuntimeError,
+                                 space.wrap("vmprof not running"))
+        name = code._get_full_name()
+        b = StringBuilder()
+        b.append('\x02')
+        write_long_to_string_builder(code._unique_id, b)
+        write_long_to_string_builder(len(name), b)
+        b.append(name)
+        os.write(self.fileno, b.build())
+
+    def disable(self, space):
+        if not self.is_enabled:
+            raise oefmt(space.w_ValueError, "_vmprof not enabled")
+        self.is_enabled = False
+        self.fileno = -1
+        if we_are_translated():
+           # does not work untranslated
+            res = vmprof_disable()
+        else:
+            res = 0
+        space.set_code_callback(None)
+        if res == -1:
+            raise wrap_oserror(space, OSError(rposix.get_saved_errno(),
+                                              "_vmprof.disable"))
+
+def vmprof_register_code(space, code):
+    from pypy.module._vmprof import Module
+    mod_vmprof = space.getbuiltinmodule('_vmprof')
+    assert isinstance(mod_vmprof, Module)
+    mod_vmprof.vmprof.register_code(space, code)
+        
+ at unwrap_spec(fileno=int, period=int)
+def enable(space, fileno, period=-1):
+    from pypy.module._vmprof import Module
+    mod_vmprof = space.getbuiltinmodule('_vmprof')
+    assert isinstance(mod_vmprof, Module)
+    mod_vmprof.vmprof.enable(space, fileno, period)
+
+def disable(space):
+    from pypy.module._vmprof import Module
+    mod_vmprof = space.getbuiltinmodule('_vmprof')
+    assert isinstance(mod_vmprof, Module)
+    mod_vmprof.vmprof.disable(space)
+
diff --git a/pypy/module/_vmprof/src/config.h b/pypy/module/_vmprof/src/config.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/src/config.h
@@ -0,0 +1,2 @@
+#define HAVE_SYS_UCONTEXT_H
+#define PC_FROM_UCONTEXT uc_mcontext.gregs[REG_RIP]
diff --git a/pypy/module/_vmprof/src/fake_pypy_api.c b/pypy/module/_vmprof/src/fake_pypy_api.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/src/fake_pypy_api.c
@@ -0,0 +1,29 @@
+
+long pypy_jit_start_addr(void)
+{
+	return 3;
+}
+
+long pypy_jit_end_addr(void)
+{
+	return 3;
+}
+
+long pypy_jit_stack_depth_at_loc(long x)
+{
+	return 0;
+}
+
+long pypy_find_codemap_at_addr(long x)
+{
+	return 0;
+}
+
+long pypy_yield_codemap_at_addr(long x, long y, long *a)
+{
+	return 0;
+}
+
+void pypy_pyframe_execute_frame(void)
+{
+}
diff --git a/pypy/module/_vmprof/src/get_custom_offset.c b/pypy/module/_vmprof/src/get_custom_offset.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/src/get_custom_offset.c
@@ -0,0 +1,46 @@
+
+long pypy_jit_start_addr();
+long pypy_jit_end_addr();
+long pypy_jit_stack_depth_at_loc(long);
+long pypy_find_codemap_at_addr(long);
+long pypy_yield_codemap_at_addr(long, long, long*);
+
+void vmprof_set_tramp_range(void* start, void* end)
+{
+}
+
+static ptrdiff_t vmprof_unw_get_custom_offset(void* ip, unw_cursor_t *cp) {
+	intptr_t ip_l = (intptr_t)ip;
+
+	if (ip_l < pypy_jit_start_addr() || ip_l > pypy_jit_end_addr()) {
+		return -1;
+	}
+	return (void*)pypy_jit_stack_depth_at_loc(ip_l);
+}
+
+static long vmprof_write_header_for_jit_addr(void **result, long n,
+											 void *ip, int max_depth)
+{
+	long codemap_pos;
+	long current_pos = 0;
+	intptr_t id;
+	intptr_t addr = (intptr_t)ip;
+
+	if (addr < pypy_jit_start_addr() || addr > pypy_jit_end_addr()) {
+		return n;
+	}
+	codemap_pos = pypy_find_codemap_at_addr(addr);
+	if (codemap_pos == -1) {
+		return n;
+	}
+	while (1) {
+		id = pypy_yield_codemap_at_addr(codemap_pos, addr, ¤t_pos);
+		if (id == 0) {
+			return n;
+		}
+		result[n++] = id;
+		if (n >= max_depth) {
+			return n;
+		}
+	}
+}
diff --git a/pypy/module/_vmprof/src/getpc.h b/pypy/module/_vmprof/src/getpc.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/src/getpc.h
@@ -0,0 +1,187 @@
+// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*-
+// Copyright (c) 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// ---
+// Author: Craig Silverstein
+//
+// This is an internal header file used by profiler.cc.  It defines
+// the single (inline) function GetPC.  GetPC is used in a signal
+// handler to figure out the instruction that was being executed when
+// the signal-handler was triggered.
+//
+// To get this, we use the ucontext_t argument to the signal-handler
+// callback, which holds the full context of what was going on when
+// the signal triggered.  How to get from a ucontext_t to a Program
+// Counter is OS-dependent.
+
+#ifndef BASE_GETPC_H_
+#define BASE_GETPC_H_
+
+#include "config.h"
+
+// On many linux systems, we may need _GNU_SOURCE to get access to
+// the defined constants that define the register we want to see (eg
+// REG_EIP).  Note this #define must come first!
+#define _GNU_SOURCE 1
+// If #define _GNU_SOURCE causes problems, this might work instead.
+// It will cause problems for FreeBSD though!, because it turns off
+// the needed __BSD_VISIBLE.
+//#define _XOPEN_SOURCE 500
+
+#include          // for memcmp
+#if defined(HAVE_SYS_UCONTEXT_H)
+#include 
+#elif defined(HAVE_UCONTEXT_H)
+#include        // for ucontext_t (and also mcontext_t)
+#elif defined(HAVE_CYGWIN_SIGNAL_H)
+#include 
+typedef ucontext ucontext_t;
+#endif
+
+
+// Take the example where function Foo() calls function Bar().  For
+// many architectures, Bar() is responsible for setting up and tearing
+// down its own stack frame.  In that case, it's possible for the
+// interrupt to happen when execution is in Bar(), but the stack frame
+// is not properly set up (either before it's done being set up, or
+// after it's been torn down but before Bar() returns).  In those
+// cases, the stack trace cannot see the caller function anymore.
+//
+// GetPC can try to identify this situation, on architectures where it
+// might occur, and unwind the current function call in that case to
+// avoid false edges in the profile graph (that is, edges that appear
+// to show a call skipping over a function).  To do this, we hard-code
+// in the asm instructions we might see when setting up or tearing
+// down a stack frame.
+//
+// This is difficult to get right: the instructions depend on the
+// processor, the compiler ABI, and even the optimization level.  This
+// is a best effort patch -- if we fail to detect such a situation, or
+// mess up the PC, nothing happens; the returned PC is not used for
+// any further processing.
+struct CallUnrollInfo {
+  // Offset from (e)ip register where this instruction sequence
+  // should be matched. Interpreted as bytes. Offset 0 is the next
+  // instruction to execute. Be extra careful with negative offsets in
+  // architectures of variable instruction length (like x86) - it is
+  // not that easy as taking an offset to step one instruction back!
+  int pc_offset;
+  // The actual instruction bytes. Feel free to make it larger if you
+  // need a longer sequence.
+  unsigned char ins[16];
+  // How many bytes to match from ins array?
+  int ins_size;
+  // The offset from the stack pointer (e)sp where to look for the
+  // call return address. Interpreted as bytes.
+  int return_sp_offset;
+};
+
+
+// The dereferences needed to get the PC from a struct ucontext were
+// determined at configure time, and stored in the macro
+// PC_FROM_UCONTEXT in config.h.  The only thing we need to do here,
+// then, is to do the magic call-unrolling for systems that support it.
+
+// -- Special case 1: linux x86, for which we have CallUnrollInfo
+#if defined(__linux) && defined(__i386) && defined(__GNUC__)
+static const CallUnrollInfo callunrollinfo[] = {
+  // Entry to a function:  push %ebp;  mov  %esp,%ebp
+  // Top-of-stack contains the caller IP.
+  { 0,
+    {0x55, 0x89, 0xe5}, 3,
+    0
+  },
+  // Entry to a function, second instruction:  push %ebp;  mov  %esp,%ebp
+  // Top-of-stack contains the old frame, caller IP is +4.
+  { -1,
+    {0x55, 0x89, 0xe5}, 3,
+    4
+  },
+  // Return from a function: RET.
+  // Top-of-stack contains the caller IP.
+  { 0,
+    {0xc3}, 1,
+    0
+  }
+};
+
+inline void* GetPC(ucontext_t *signal_ucontext) {
+  // See comment above struct CallUnrollInfo.  Only try instruction
+  // flow matching if both eip and esp looks reasonable.
+  const int eip = signal_ucontext->uc_mcontext.gregs[REG_EIP];
+  const int esp = signal_ucontext->uc_mcontext.gregs[REG_ESP];
+  if ((eip & 0xffff0000) != 0 && (~eip & 0xffff0000) != 0 &&
+      (esp & 0xffff0000) != 0) {
+    char* eip_char = reinterpret_cast(eip);
+    for (int i = 0; i < sizeof(callunrollinfo)/sizeof(*callunrollinfo); ++i) {
+      if (!memcmp(eip_char + callunrollinfo[i].pc_offset,
+                  callunrollinfo[i].ins, callunrollinfo[i].ins_size)) {
+        // We have a match.
+        void **retaddr = (void**)(esp + callunrollinfo[i].return_sp_offset);
+        return *retaddr;
+      }
+    }
+  }
+  return (void*)eip;
+}
+
+// Special case #2: Windows, which has to do something totally different.
+#elif defined(_WIN32) || defined(__CYGWIN__) || defined(__CYGWIN32__) || defined(__MINGW32__)
+// If this is ever implemented, probably the way to do it is to have
+// profiler.cc use a high-precision timer via timeSetEvent:
+//    http://msdn2.microsoft.com/en-us/library/ms712713.aspx
+// We'd use it in mode TIME_CALLBACK_FUNCTION/TIME_PERIODIC.
+// The callback function would be something like prof_handler, but
+// alas the arguments are different: no ucontext_t!  I don't know
+// how we'd get the PC (using StackWalk64?)
+//    http://msdn2.microsoft.com/en-us/library/ms680650.aspx
+
+#include "base/logging.h"   // for RAW_LOG
+#ifndef HAVE_CYGWIN_SIGNAL_H
+typedef int ucontext_t;
+#endif
+
+inline void* GetPC(ucontext_t *signal_ucontext) {
+  RAW_LOG(ERROR, "GetPC is not yet implemented on Windows\n");
+  return NULL;
+}
+
+// Normal cases.  If this doesn't compile, it's probably because
+// PC_FROM_UCONTEXT is the empty string.  You need to figure out
+// the right value for your system, and add it to the list in
+// configure.ac (or set it manually in your config.h).
+#else
+inline void* GetPC(ucontext_t *signal_ucontext) {
+  return (void*)signal_ucontext->PC_FROM_UCONTEXT;   // defined in config.h
+}
+
+#endif
+
+#endif  // BASE_GETPC_H_
diff --git a/pypy/module/_vmprof/src/trampoline.asmgcc.s b/pypy/module/_vmprof/src/trampoline.asmgcc.s
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/src/trampoline.asmgcc.s
@@ -0,0 +1,17 @@
+// NOTE: you need to use TABs, not spaces!
+        
+	.text
+	.p2align 4,,-1
+	.globl	pypy_execute_frame_trampoline
+	.type	pypy_execute_frame_trampoline, @function
+pypy_execute_frame_trampoline:
+	.cfi_startproc
+	pushq	%rdi
+	.cfi_def_cfa_offset 16
+	call pypy_pyframe_execute_frame at PLT
+	/* GCROOT 0(%rsp) */
+	popq	%rdi
+	.cfi_def_cfa_offset 8
+	ret
+	.cfi_endproc
+	.size	pypy_execute_frame_trampoline, .-pypy_execute_frame_trampoline
diff --git a/pypy/module/_vmprof/src/trampoline.h b/pypy/module/_vmprof/src/trampoline.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/src/trampoline.h
@@ -0,0 +1,1 @@
+void* pypy_execute_frame_trampoline(void*, void*, void*);
diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/src/vmprof.c
@@ -0,0 +1,340 @@
+/* VMPROF
+ *
+ * statistical sampling profiler specifically designed to profile programs
+ * which run on a Virtual Machine and/or bytecode interpreter, such as Python,
+ * etc.
+ *
+ * The logic to dump the C stack traces is partly stolen from the code in gperftools.
+ * The file "getpc.h" has been entirely copied from gperftools.
+ *
+ * Tested only on gcc, linux, x86_64.
+ *
+ * Copyright (C) 2014-2015
+ *   Antonio Cuni - anto.cuni at gmail.com
+ *   Maciej Fijalkowski - fijall at gmail.com
+ *
+ */
+
+
+#include "getpc.h"      // should be first to get the _GNU_SOURCE dfn
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#define UNW_LOCAL_ONLY
+#include 
+
+#include "vmprof.h"
+
+#define _unused(x) ((void)x)
+
+#define MAX_FUNC_NAME 128
+#define MAX_STACK_DEPTH 64
+
+static FILE* profile_file = NULL;
+void* vmprof_mainloop_func;
+static ptrdiff_t mainloop_sp_offset;
+static vmprof_get_virtual_ip_t mainloop_get_virtual_ip;
+
+
+/* *************************************************************
+ * functions to write a profile file compatible with gperftools
+ * *************************************************************
+ */
+
+#define MARKER_STACKTRACE '\x01'
+#define MARKER_VIRTUAL_IP '\x02'
+#define MARKER_TRAILER '\x03'
+
+static void prof_word(FILE* f, long x) {
+    fwrite(&x, sizeof(x), 1, f);
+}
+
+static void prof_header(FILE* f, long period_usec) {
+    prof_word(f, 0);
+    prof_word(f, 3);
+    prof_word(f, 0);
+    prof_word(f, period_usec);
+    prof_word(f, 0);
+}
+
+static void prof_write_stacktrace(FILE* f, void** stack, int depth, int count) {
+    int i;
+	char marker = MARKER_STACKTRACE;
+
+	fwrite(&marker, 1, 1, f);
+    prof_word(f, count);
+    prof_word(f, depth);
+    for(i=0; isp = bp;
+		bp -= sizeof(void*);
+        cp2->ip = ((void**)bp)[0];
+        // the ret is on the top of the stack minus WORD
+        return 1;
+    }
+}
+
+
+/* *************************************************************
+ * functions to dump the stack trace
+ * *************************************************************
+ */
+
+// stolen from pprof:
+// Sometimes, we can try to get a stack trace from within a stack
+// trace, because libunwind can call mmap (maybe indirectly via an
+// internal mmap based memory allocator), and that mmap gets trapped
+// and causes a stack-trace request.  If were to try to honor that
+// recursive request, we'd end up with infinite recursion or deadlock.
+// Luckily, it's safe to ignore those subsequent traces.  In such
+// cases, we return 0 to indicate the situation.
+//static __thread int recursive;
+static int recursive; // XXX antocuni: removed __thread
+
+int get_stack_trace(void** result, int max_depth, ucontext_t *ucontext) {
+    void *ip;
+    int n = 0;
+    unw_cursor_t cursor;
+    unw_context_t uc = *ucontext;
+    if (recursive) {
+        return 0;
+    }
+    ++recursive;
+
+    int ret = unw_init_local(&cursor, &uc);
+    assert(ret >= 0);
+    _unused(ret);
+
+    while (n < max_depth) {
+        if (unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *) &ip) < 0) {
+            break;
+        }
+
+        unw_proc_info_t pip;
+        unw_get_proc_info(&cursor, &pip);
+
+        /* char funcname[4096]; */
+        /* unw_word_t offset; */
+        /* unw_get_proc_name(&cursor, funcname, 4096, &offset); */
+        /* printf("%s+%#lx <%p>\n", funcname, offset, ip); */
+
+        /* if n==0, it means that the signal handler interrupted us while we
+           were in the trampoline, so we are not executing (yet) the real main
+           loop function; just skip it */
+        if (vmprof_mainloop_func && 
+            (void*)pip.start_ip == (void*)vmprof_mainloop_func &&
+            n > 0) {
+          // found main loop stack frame
+          void* sp;
+          unw_get_reg(&cursor, UNW_REG_SP, (unw_word_t *) &sp);
+          void *arg_addr = (char*)sp + mainloop_sp_offset;
+          void **arg_ptr = (void**)arg_addr;
+          // fprintf(stderr, "stacktrace mainloop: rsp %p   &f2 %p   offset %ld\n", 
+          //         sp, arg_addr, mainloop_sp_offset);
+          ip = mainloop_get_virtual_ip(*arg_ptr);
+        }
+
+        result[n++] = ip;
+		n = vmprof_write_header_for_jit_addr(result, n, ip, max_depth);
+        if (vmprof_unw_step(&cursor) <= 0) {
+            break;
+        }
+    }
+    --recursive;
+    return n;
+}
+
+
+static int __attribute__((noinline)) frame_forcer(int rv) {
+    return rv;
+}
+
+static void sigprof_handler(int sig_nr, siginfo_t* info, void *ucontext) {
+    void* stack[MAX_STACK_DEPTH];
+    stack[0] = GetPC((ucontext_t*)ucontext);
+    int depth = frame_forcer(get_stack_trace(stack+1, MAX_STACK_DEPTH-1, ucontext));
+    depth++;  // To account for pc value in stack[0];
+    prof_write_stacktrace(profile_file, stack, depth, 1);
+}
+
+/* *************************************************************
+ * functions to enable/disable the profiler
+ * *************************************************************
+ */
+
+static int open_profile(int fd, long period_usec, int write_header, char *s,
+						int slen) {
+	if ((fd = dup(fd)) == -1) {
+		return -1;
+	}
+    profile_file = fdopen(fd, "wb");
+	if (!profile_file) {
+		return -1;
+	}
+	if (write_header)
+		prof_header(profile_file, period_usec);
+	if (s)
+		fwrite(s, slen, 1, profile_file);
+	return 0;
+}
+
+static int close_profile(void) {
+	// XXX all of this can happily fail
+    FILE* src;
+    char buf[BUFSIZ];
+    size_t size;
+	int marker = MARKER_TRAILER;
+	fwrite(&marker, 1, 1, profile_file);
+
+    // copy /proc/PID/maps to the end of the profile file
+    sprintf(buf, "/proc/%d/maps", getpid());
+    src = fopen(buf, "r");    
+    while ((size = fread(buf, 1, BUFSIZ, src))) {
+        fwrite(buf, 1, size, profile_file);
+    }
+    fclose(src);
+    fclose(profile_file);
+	return 0;
+}
+
+
+static int install_sigprof_handler(void) {
+    struct sigaction sa;
+    memset(&sa, 0, sizeof(sa));
+    sa.sa_sigaction = sigprof_handler;
+    sa.sa_flags = SA_RESTART | SA_SIGINFO;
+    if (sigemptyset(&sa.sa_mask) == -1 ||
+		sigaction(SIGPROF, &sa, NULL) == -1) {
+		return -1;
+	}
+	return 0;
+}
+
+static int remove_sigprof_handler(void) {
+    //sighandler_t res = signal(SIGPROF, SIG_DFL);
+	//if (res == SIG_ERR) {
+	//	return -1;
+	//}
+	return 0;
+};
+
+static int install_sigprof_timer(long period_usec) {
+    static struct itimerval timer;
+    timer.it_interval.tv_sec = 0;
+    timer.it_interval.tv_usec = period_usec;
+    timer.it_value = timer.it_interval;
+    if (setitimer(ITIMER_PROF, &timer, NULL) != 0) {
+		return -1;
+    }
+	return 0;
+}
+
+static int remove_sigprof_timer(void) {
+    static struct itimerval timer;
+    timer.it_interval.tv_sec = 0;
+    timer.it_interval.tv_usec = 0;
+    timer.it_value = timer.it_interval;
+    if (setitimer(ITIMER_PROF, &timer, NULL) != 0) {
+		return -1;
+    }
+	return 0;
+}
+
+/* *************************************************************
+ * public API
+ * *************************************************************
+ */
+
+void vmprof_set_mainloop(void* func, ptrdiff_t sp_offset, 
+                         vmprof_get_virtual_ip_t get_virtual_ip) {
+    mainloop_sp_offset = sp_offset;
+    mainloop_get_virtual_ip = get_virtual_ip;
+    vmprof_mainloop_func = func;
+}
+
+int vmprof_enable(int fd, long period_usec, int write_header, char *s,
+				  int slen)
+{
+    if (period_usec == -1)
+        period_usec = 1000000 / 100; /* 100hz */
+    if (open_profile(fd, period_usec, write_header, s, slen) == -1) {
+		return -1;
+	}
+    if (install_sigprof_handler() == -1) {
+		return -1;
+	}
+    if (install_sigprof_timer(period_usec) == -1) {
+		return -1;
+	}
+	return 0;
+}
+
+int vmprof_disable(void) {
+    if (remove_sigprof_timer() == -1) {
+		return -1;
+	}
+    if (remove_sigprof_handler() == -1) {
+		return -1;
+	}
+    if (close_profile() == -1) {
+		return -1;
+	}
+	return 0;
+}
+
+void vmprof_register_virtual_function(const char* name, void* start, void* end) {
+    // for now *end is simply ignored
+	char buf[1024];
+	int lgt = strlen(name) + 2 * sizeof(long) + 1;
+
+	if (lgt > 1024) {
+		lgt = 1024;
+	}
+	buf[0] = MARKER_VIRTUAL_IP;
+	((void **)(((void*)buf) + 1))[0] = start;
+	((long *)(((void*)buf) + 1 + sizeof(long)))[0] = lgt - 2 * sizeof(long) - 1;
+	strncpy(buf + 2 * sizeof(long) + 1, name, 1024 - 2 * sizeof(long) - 1);
+	fwrite(buf, lgt, 1, profile_file);
+}
diff --git a/pypy/module/_vmprof/src/vmprof.h b/pypy/module/_vmprof/src/vmprof.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/src/vmprof.h
@@ -0,0 +1,22 @@
+#ifndef VMPROF_VMPROF_H_
+#define VMPROF_VMPROF_H_
+
+#include 
+
+typedef void* (*vmprof_get_virtual_ip_t)(void*);
+
+extern void* vmprof_mainloop_func;
+void vmprof_set_mainloop(void* func, ptrdiff_t sp_offset, 
+                         vmprof_get_virtual_ip_t get_virtual_ip);
+
+void vmprof_register_virtual_function(const char* name, void* start, void* end);
+
+
+int vmprof_enable(int fd, long period_usec, int write_header, char* vips,
+				  int vips_len);
+int vmprof_disable(void);
+
+// XXX: this should be part of _vmprof (the CPython extension), not vmprof (the library)
+void vmprof_set_tramp_range(void* start, void* end);
+
+#endif
diff --git a/pypy/module/_vmprof/test/__init__.py b/pypy/module/_vmprof/test/__init__.py
new file mode 100644
diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/test/test__vmprof.py
@@ -0,0 +1,55 @@
+
+import tempfile
+from pypy.tool.pytest.objspace import gettestobjspace
+
+class AppTestVMProf(object):
+    def setup_class(cls):
+        cls.space = gettestobjspace(usemodules=['_vmprof', 'struct'])
+        cls.tmpfile = tempfile.NamedTemporaryFile()
+        cls.w_tmpfileno = cls.space.wrap(cls.tmpfile.fileno())
+        cls.w_tmpfilename = cls.space.wrap(cls.tmpfile.name)
+        cls.tmpfile2 = tempfile.NamedTemporaryFile()
+        cls.w_tmpfileno2 = cls.space.wrap(cls.tmpfile2.fileno())
+        cls.w_tmpfilename2 = cls.space.wrap(cls.tmpfile2.name)
+
+    def test_import_vmprof(self):
+        import struct, sys
+
+        WORD = struct.calcsize('l')
+        
+        def count(s):
+            i = 0
+            count = 0
+            i += 5 * WORD # header
+            while i < len(s):
+                assert s[i] == '\x02'
+                i += 1
+                _, size = struct.unpack("ll", s[i:i + 2 * WORD])
+                count += 1
+                i += 2 * WORD + size
+            return count
+        
+        import _vmprof
+        _vmprof.enable(self.tmpfileno)
+        _vmprof.disable()
+        s = open(self.tmpfilename).read()
+        no_of_codes = count(s)
+        assert no_of_codes > 10
+        d = {}
+
+        exec """def foo():
+            pass
+        """ in d
+
+        _vmprof.enable(self.tmpfileno2)
+
+        exec """def foo2():
+            pass
+        """ in d
+
+        _vmprof.disable()
+        s = open(self.tmpfilename2).read()
+        no_of_codes2 = count(s)
+        assert "py:foo:" in s
+        assert "py:foo2:" in s
+        assert no_of_codes2 >= no_of_codes + 2 # some extra codes from tests
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -35,6 +35,9 @@
     name = opcode_method_names[ord(bytecode.co_code[next_instr])]
     return '%s #%d %s' % (bytecode.get_repr(), next_instr, name)
 
+def get_unique_id(next_instr, is_being_profiled, bytecode):
+    return bytecode._unique_id
+
 
 def should_unroll_one_iteration(next_instr, is_being_profiled, bytecode):
     return (bytecode.co_flags & CO_GENERATOR) != 0
@@ -45,6 +48,7 @@
     virtualizables = ['frame']
 
 pypyjitdriver = PyPyJitDriver(get_printable_location = get_printable_location,
+                              get_unique_id = get_unique_id,
                               should_unroll_one_iteration =
                               should_unroll_one_iteration,
                               name='pypyjit')
diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -57,6 +57,7 @@
         BaseAssembler.setup_once(self)
 
     def setup(self, looptoken):
+        BaseAssembler.setup(self, looptoken)
         assert self.memcpy_addr != 0, 'setup_once() not called?'
         if we_are_translated():
             self.debug = False
@@ -71,7 +72,6 @@
         self.mc.datablockwrapper = self.datablockwrapper
         self.target_tokens_currently_compiling = {}
         self.frame_depth_to_patch = []
-        self._finish_gcmap = lltype.nullptr(jitframe.GCMAP)
 
     def teardown(self):
         self.current_clt = None
@@ -659,6 +659,7 @@
             assert len(set(inputargs)) == len(inputargs)
 
         self.setup(original_loop_token)
+        self.codemap.inherit_code_from_position(faildescr.adr_jump_offset)
         descr_number = compute_unique_id(faildescr)
         if log:
             operations = self._inject_debugging_code(faildescr, operations,
@@ -879,8 +880,12 @@
         self.datablockwrapper.done()      # finish using cpu.asmmemmgr
         self.datablockwrapper = None
         allblocks = self.get_asmmemmgr_blocks(looptoken)
-        return self.mc.materialize(self.cpu.asmmemmgr, allblocks,
+        size = self.mc.get_relative_pos() 
+        res = self.mc.materialize(self.cpu.asmmemmgr, allblocks,
                                    self.cpu.gc_ll_descr.gcrootmap)
+        self.cpu.asmmemmgr.register_codemap(
+            self.codemap.get_final_bytecode(res, size))
+        return res
 
     def update_frame_depth(self, frame_depth):
         baseofs = self.cpu.get_baseofs_of_frame_field()
diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -392,6 +392,9 @@
         else:
             self.rm._sync_var(v)
 
+    def prepare_op_debug_merge_point(self, op, fcond):
+        self.assembler.codemap.debug_merge_point(op)
+
     def _prepare_op_int_add(self, op, fcond):
         boxes = op.getarglist()
         a0, a1 = boxes
diff --git a/rpython/jit/backend/llsupport/asmmemmgr.py b/rpython/jit/backend/llsupport/asmmemmgr.py
--- a/rpython/jit/backend/llsupport/asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/asmmemmgr.py
@@ -4,7 +4,10 @@
 from rpython.rlib import rmmap
 from rpython.rlib.debug import debug_start, debug_print, debug_stop
 from rpython.rlib.debug import have_debug_prints
-from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rlib.rbisect import bisect, bisect_tuple
+
+_memmngr = None # global reference so we can use @entrypoint :/
 
 
 class AsmMemoryManager(object):
@@ -24,6 +27,12 @@
         self.free_blocks = {}      # map {start: stop}
         self.free_blocks_end = {}  # map {stop: start}
         self.blocks_by_size = [[] for i in range(self.num_indices)]
+        # two lists of jit addresses (sorted) and the corresponding stack
+        # depths
+        self.jit_addr_map = []
+        self.jit_frame_depth_map = []
+        self.jit_codemap = []
+        # see codemap.py
 
     def malloc(self, minsize, maxsize):
         """Allocate executable memory, between minsize and maxsize bytes,
@@ -45,6 +54,13 @@
         if r_uint is not None:
             self.total_mallocs -= r_uint(stop - start)
         self._add_free_block(start, stop)
+        # fix up jit_addr_map
+        jit_adr_start = bisect(self.jit_addr_map, start)
+        jit_adr_stop = bisect(self.jit_addr_map, stop)
+        self.jit_addr_map = (self.jit_addr_map[:jit_adr_start] +
+                             self.jit_addr_map[jit_adr_stop:])
+        self.jit_frame_depth_map = (self.jit_frame_depth_map[:jit_adr_start] +
+                                    self.jit_frame_depth_map[jit_adr_stop:])
 
     def open_malloc(self, minsize):
         """Allocate at least minsize bytes.  Returns (start, stop)."""
@@ -151,6 +167,35 @@
         del self.free_blocks_end[stop]
         return (start, stop)
 
+    def register_frame_depth_map(self, rawstart, frame_positions,
+                                 frame_assignments):
+        if not frame_positions:
+            return
+        if not self.jit_addr_map or rawstart > self.jit_addr_map[-1]:
+            start = len(self.jit_addr_map)
+            self.jit_addr_map += [0] * len(frame_positions)
+            self.jit_frame_depth_map += [0] * len(frame_positions)
+        else:
+            start = bisect(self.jit_addr_map, rawstart)
+            self.jit_addr_map = (self.jit_addr_map[:start] +
+                                 [0] * len(frame_positions) +
+                                 self.jit_addr_map[start:])
+            self.jit_frame_depth_map = (self.jit_frame_depth_map[:start] +
+                                 [0] * len(frame_positions) +
+                                 self.jit_frame_depth_map[start:])
+        for i, pos in enumerate(frame_positions):
+            self.jit_addr_map[i + start] = pos + rawstart
+            self.jit_frame_depth_map[i + start] = frame_assignments[i]
+
+    def register_codemap(self, codemap):
+        start = codemap[0]
+        pos = bisect_tuple(self.jit_codemap, start)
+        if pos == len(self.jit_codemap): # common case
+            self.jit_codemap.append(codemap)
+        else:
+            self.jit_codemap = (self.jit_codemap[:pos] + [codemap] +
+                                self.jit_codemap[pos:])
+
     def _delete(self):
         "NOT_RPYTHON"
         if self._allocated:
@@ -212,6 +257,9 @@
 
     gcroot_markers = None
 
+    frame_positions = None
+    frame_assignments = None
+
     def __init__(self, translated=None):
         if translated is None:
             translated = we_are_translated()
@@ -316,6 +364,10 @@
             assert gcrootmap is not None
             for pos, mark in self.gcroot_markers:
                 gcrootmap.register_asm_addr(rawstart + pos, mark)
+        asmmemmgr.register_frame_depth_map(rawstart, self.frame_positions,
+                                           self.frame_assignments)
+        self.frame_positions = None
+        self.frame_assignments = None
         return rawstart
 
     def _become_a_plain_block_builder(self):
diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -1,6 +1,7 @@
 from rpython.jit.backend.llsupport import jitframe
 from rpython.jit.backend.llsupport.memcpy import memcpy_fn, memset_fn
 from rpython.jit.backend.llsupport.symbolic import WORD
+from rpython.jit.backend.llsupport.codemap import CodemapBuilder
 from rpython.jit.metainterp.history import (INT, REF, FLOAT, JitCellToken,
     ConstInt, BoxInt, AbstractFailDescr)
 from rpython.jit.metainterp.resoperation import ResOperation, rop
@@ -128,6 +129,10 @@
                                               track_allocation=False)
         self.gcmap_for_finish[0] = r_uint(1)
 
+    def setup(self, looptoken):
+        self.codemap = CodemapBuilder()
+        self._finish_gcmap = lltype.nullptr(jitframe.GCMAP)
+
     def set_debug(self, v):
         r = self._debug
         self._debug = v
@@ -194,6 +199,9 @@
         guardtok.faildescr.rd_locs = positions
         return fail_descr, target
 
+    def debug_merge_point(self, op):
+        self.codemap.debug_merge_point(op, self.mc.get_relative_pos())
+
     def call_assembler(self, op, guard_op, argloc, vloc, result_loc, tmploc):
         self._store_force_index(guard_op)
         descr = op.getdescr()
@@ -276,6 +284,9 @@
         # YYY very minor leak -- we need the counters to stay alive
         # forever, just because we want to report them at the end
         # of the process
+
+        # XXX the numbers here are ALMOST unique, but not quite, use a counter
+        #     or something
         struct = lltype.malloc(DEBUG_COUNTER, flavor='raw',
                                track_allocation=False)
         struct.i = 0
diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/llsupport/codemap.py
@@ -0,0 +1,142 @@
+
+""" Bytecode for storage in asmmemmgr.jit_codemap. Format is as follows:
+
+ list of tuples of shape (addr, machine code size, bytecode info)
+ where bytecode info is a string made up of:
+    8 bytes unique_id, 4 bytes start_addr (relative), 4 bytes size (relative),
+    2 bytes how many items to skip to go to the next on similar level
+    [so far represented by a list of integers for simplicity]
+
+"""
+
+from rpython.rlib import rgc
+from rpython.rlib.entrypoint import jit_entrypoint
+from rpython.jit.backend.llsupport import asmmemmgr
+from rpython.rlib.rbisect import bisect, bisect_tuple
+from rpython.rtyper.lltypesystem import lltype, rffi
+
+ at jit_entrypoint([lltype.Signed], lltype.Signed,
+                c_name='pypy_jit_stack_depth_at_loc')
+ at rgc.no_collect
+def stack_depth_at_loc(loc):
+    _memmngr = asmmemmgr._memmngr
+
+    pos = bisect(_memmngr.jit_addr_map, loc)
+    if pos == 0 or pos == len(_memmngr.jit_addr_map):
+        return -1
+    return _memmngr.jit_frame_depth_map[pos-1]
+
+ at jit_entrypoint([], lltype.Signed, c_name='pypy_jit_start_addr')
+def jit_start_addr():
+    _memmngr = asmmemmgr._memmngr
+
+    return _memmngr.jit_addr_map[0]
+
+ at jit_entrypoint([], lltype.Signed, c_name='pypy_jit_end_addr')
+def jit_end_addr():
+    _memmngr = asmmemmgr._memmngr
+
+    return _memmngr.jit_addr_map[-1]
+
+ at jit_entrypoint([lltype.Signed], lltype.Signed,
+                c_name='pypy_find_codemap_at_addr')
+def find_codemap_at_addr(addr):
+    _memmngr = asmmemmgr._memmngr
+
+    res = bisect_tuple(_memmngr.jit_codemap, addr) - 1
+    if res == len(_memmngr.jit_codemap):
+        return -1
+    return res
+
+ at jit_entrypoint([lltype.Signed, lltype.Signed,
+                 rffi.CArrayPtr(lltype.Signed)], lltype.Signed,
+                 c_name='pypy_yield_codemap_at_addr')
+def yield_bytecode_at_addr(codemap_no, addr, current_pos_addr):
+    """ will return consecutive unique_ids from codemap, starting from position
+    `pos` until addr
+    """
+    _memmngr = asmmemmgr._memmngr
+
+    codemap = _memmngr.jit_codemap[codemap_no]
+    current_pos = current_pos_addr[0]
+    start_addr = codemap[0]
+    rel_addr = addr - start_addr
+    while True:
+        if current_pos >= len(codemap[2]):
+            return 0
+        next_start = codemap[2][current_pos + 1]
+        if next_start > rel_addr:
+            return 0
+        next_stop = codemap[2][current_pos + 2]
+        if next_stop > rel_addr:
+            current_pos_addr[0] = current_pos + 4
+            return codemap[2][current_pos]
+        # we need to skip potentially more than one
+        current_pos = codemap[2][current_pos + 3]
+
+def unpack_traceback(addr):
+    codemap_pos = find_codemap_at_addr(addr)
+    assert codemap_pos >= 0
+    storage = lltype.malloc(rffi.CArray(lltype.Signed), 1, flavor='raw')
+    storage[0] = 0
+    res = []
+    while True:
+        item = yield_bytecode_at_addr(codemap_pos, addr, storage)
+        if item == 0:
+            break
+        res.append(item)
+    lltype.free(storage, flavor='raw')
+    return res
+
+
+class CodemapBuilder(object):
+    def __init__(self):
+        self.l = []
+        self.patch_position = []
+        self.last_call_depth = -1
+
+    def debug_merge_point(self, op, pos):
+        call_depth = op.getarg(1).getint()
+        if call_depth != self.last_call_depth:
+            unique_id = op.getarg(3).getint()
+            if unique_id == 0: # uninteresting case
+                return
+            assert unique_id & 1 == 0
+            if call_depth > self.last_call_depth:
+                self.l.append(unique_id)
+                self.l.append(pos) # <- this is a relative pos
+                self.patch_position.append(len(self.l))
+                self.l.append(0) # marker
+                self.l.append(0) # second marker
+            else:
+                for i in range(self.last_call_depth - call_depth):
+                    to_patch = self.patch_position.pop()
+                    self.l[to_patch] = pos
+                    self.l[to_patch + 1] = len(self.l)
+            self.last_call_depth = call_depth
+
+    def inherit_code_from_position(self, pos):
+        lst = unpack_traceback(pos)
+        self.last_call_depth = len(lst) - 1
+        for item in lst:
+            self.l.append(item)
+            self.l.append(0)
+            self.patch_position.append(len(self.l))
+            self.l.append(0) # marker
+            self.l.append(0) # second marker
+
+    def get_final_bytecode(self, addr, size):
+        while self.patch_position:
+            pos = self.patch_position.pop()
+            self.l[pos] = size
+            self.l[pos + 1] = len(self.l)
+        # at the end there should be no zeros
+        for i in range(len(self.l) / 4):
+            item = self.l[i * 4] # unique_id
+            assert item > 0 # no zeros here
+            item = self.l[i * 4 + 2] # end in asm
+            assert item > 0
+            item = self.l[i * 4 + 3] # end in l
+            assert item > 0
+        return (addr, size, self.l) # XXX compact self.l
+
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -16,7 +16,7 @@
     FieldDescr, ArrayDescr, CallDescr, InteriorFieldDescr,
     FLAG_POINTER, FLAG_FLOAT)
 from rpython.jit.backend.llsupport.memcpy import memset_fn
-from rpython.jit.backend.llsupport.asmmemmgr import AsmMemoryManager
+from rpython.jit.backend.llsupport import asmmemmgr
 from rpython.rlib.unroll import unrolling_iterable
 
 
@@ -48,7 +48,8 @@
             self._setup_exception_handling_translated()
         else:
             self._setup_exception_handling_untranslated()
-        self.asmmemmgr = AsmMemoryManager()
+        self.asmmemmgr = asmmemmgr.AsmMemoryManager()
+        asmmemmgr._memmngr = self.asmmemmgr
         self._setup_frame_realloc(translate_support_code)
         ad = self.gc_ll_descr.getframedescrs(self).arraydescr
         self.signedarraydescr = ad
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -17,8 +17,6 @@
 class GcRewriterAssembler(object):
     """ This class performs the following rewrites on the list of operations:
 
-     - Remove the DEBUG_MERGE_POINTs.
-
      - Turn all NEW_xxx to either a CALL_MALLOC_GC, or a CALL_MALLOC_NURSERY
        followed by SETFIELDs in order to initialize their GC fields.  The
        two advantages of CALL_MALLOC_NURSERY is that it inlines the common
@@ -60,8 +58,6 @@
         #
         for i in range(len(operations)):
             op = operations[i]
-            if op.getopnum() == rop.DEBUG_MERGE_POINT:
-                continue
             # ---------- turn NEWxxx into CALL_MALLOC_xxx ----------
             if op.is_malloc():
                 self.handle_malloc_operation(op)
diff --git a/rpython/jit/backend/llsupport/test/test_asmmemmgr.py b/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
--- a/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
@@ -2,6 +2,7 @@
 from rpython.jit.backend.llsupport.asmmemmgr import AsmMemoryManager
 from rpython.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
 from rpython.jit.backend.llsupport.asmmemmgr import BlockBuilderMixin
+from rpython.jit.backend.llsupport import asmmemmgr
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib import debug
 
@@ -157,6 +158,7 @@
         class FakeGcRootMap:
             def register_asm_addr(self, retaddr, mark):
                 puts.append((retaddr, mark))
+
         #
         mc = BlockBuilderMixin()
         mc.writechar('X')
@@ -266,3 +268,16 @@
     md.done()
     assert allblocks == [(1597, 1697), (1797, 1835)]
     assert ops == [('free', 1835, 1897)]
+
+def test_find_jit_frame_depth():
+    mgr = AsmMemoryManager()
+    mgr.register_frame_depth_map(11, [0, 5, 10], [1, 2, 3])
+    mgr.register_frame_depth_map(30, [0, 5, 10], [4, 5, 6])
+    mgr.register_frame_depth_map(0, [0, 5, 10], [7, 8, 9])
+    asmmemmgr._memmngr = mgr
+    assert asmmemmgr.stack_depth_at_loc(13) == 1
+    assert asmmemmgr.stack_depth_at_loc(-3) == -1
+    assert asmmemmgr.stack_depth_at_loc(41) == -1
+    assert asmmemmgr.stack_depth_at_loc(5) == 8
+    assert asmmemmgr.stack_depth_at_loc(17) == 2
+    assert asmmemmgr.stack_depth_at_loc(38) == 5
diff --git a/rpython/jit/backend/llsupport/test/ztranslation_test.py b/rpython/jit/backend/llsupport/test/ztranslation_test.py
--- a/rpython/jit/backend/llsupport/test/ztranslation_test.py
+++ b/rpython/jit/backend/llsupport/test/ztranslation_test.py
@@ -2,7 +2,7 @@
 from rpython.tool.udir import udir
 from rpython.rlib.jit import JitDriver, unroll_parameters, set_param
 from rpython.rlib.jit import PARAMETERS, dont_look_inside
-from rpython.rlib.jit import promote
+from rpython.rlib.jit import promote, _get_virtualizable_token
 from rpython.rlib import jit_hooks, rposix
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib.rthread import ThreadLocalReference, ThreadLocalField
@@ -28,12 +28,15 @@
         # - floats neg and abs
         # - llexternal with macro=True
 
-        class Frame(object):
+        class BasicFrame(object):
             _virtualizable_ = ['i']
 
             def __init__(self, i):
                 self.i = i
 
+        class Frame(BasicFrame):
+            pass
+
         eci = ExternalCompilationInfo(post_include_bits=['''
 #define pypy_my_fabs(x)  fabs(x)
 '''])
@@ -59,6 +62,7 @@
             while frame.i > 3:
                 jitdriver.can_enter_jit(frame=frame, total=total, j=j)
                 jitdriver.jit_merge_point(frame=frame, total=total, j=j)
+                _get_virtualizable_token(frame)
                 total += frame.i
                 if frame.i >= 20:
                     frame.i -= 2
diff --git a/rpython/jit/backend/x86/arch.py b/rpython/jit/backend/x86/arch.py
--- a/rpython/jit/backend/x86/arch.py
+++ b/rpython/jit/backend/x86/arch.py
@@ -51,3 +51,6 @@
     THREADLOCAL_OFS = (FRAME_FIXED_SIZE - 1) * WORD
 
 assert PASS_ON_MY_FRAME >= 12       # asmgcc needs at least JIT_USE_WORDS + 3
+
+# return address, followed by FRAME_FIXED_SIZE words
+DEFAULT_FRAME_BYTES = (1 + FRAME_FIXED_SIZE) * WORD
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -18,7 +18,8 @@
 from rpython.jit.backend.llsupport.regalloc import (get_scale, valid_addressing_size)
 from rpython.jit.backend.x86.arch import (FRAME_FIXED_SIZE, WORD, IS_X86_64,
                                        JITFRAME_FIXED_SIZE, IS_X86_32,
-                                       PASS_ON_MY_FRAME, THREADLOCAL_OFS)
+                                       PASS_ON_MY_FRAME, THREADLOCAL_OFS,
+                                       DEFAULT_FRAME_BYTES)
 from rpython.jit.backend.x86.regloc import (eax, ecx, edx, ebx, esp, ebp, esi,
     xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, r8, r9, r10, r11, edi,
     r12, r13, r14, r15, X86_64_SCRATCH_REG, X86_64_XMM_SCRATCH_REG,
@@ -66,6 +67,7 @@
             self._build_float_constants()
 
     def setup(self, looptoken):
+        BaseAssembler.setup(self, looptoken)
         assert self.memcpy_addr != 0, "setup_once() not called?"
         self.current_clt = looptoken.compiled_loop_token
         self.pending_guard_tokens = []
@@ -80,7 +82,6 @@
                                                         allblocks)
         self.target_tokens_currently_compiling = {}
         self.frame_depth_to_patch = []
-        self._finish_gcmap = lltype.nullptr(jitframe.GCMAP)
 
     def teardown(self):
         self.pending_guard_tokens = None
@@ -267,6 +268,10 @@
         # the correct "ret" arg
         offset = mc.get_relative_pos() - jz_location
         mc.overwrite32(jz_location-4, offset)
+        # From now on this function is basically "merged" with
+        # its caller and so contains DEFAULT_FRAME_BYTES bytes
+        # plus my own return address, which we'll ignore next
+        mc.force_frame_size(DEFAULT_FRAME_BYTES + WORD)
         mc.ADD_ri(esp.value, WORD)
         mc.JMP(imm(self.propagate_exception_path))
         #
@@ -278,6 +283,7 @@
             return      # not supported (for tests, or non-translated)
         #
         self.mc = codebuf.MachineCodeBlockWrapper()
+        self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         #
         # read and reset the current exception
 
@@ -331,7 +337,10 @@
         offset = mc.get_relative_pos() - jnz_location
         assert 0 < offset <= 127
         mc.overwrite(jnz_location-1, chr(offset))
-        # adjust the esp to point back to the previous return
+        # From now on this function is basically "merged" with
+        # its caller and so contains DEFAULT_FRAME_BYTES bytes
+        # plus my own return address, which we'll ignore next
+        mc.force_frame_size(DEFAULT_FRAME_BYTES + WORD)
         mc.ADD_ri(esp.value, WORD)
         mc.JMP(imm(self.propagate_exception_path))
         #
@@ -409,6 +418,8 @@
                 mc.LEA_rs(esp.value, 2 * WORD)
             self._pop_all_regs_from_frame(mc, [], withfloats, callee_only=True)
             mc.RET16_i(WORD)
+            # Note that wb_slowpath[0..3] end with a RET16_i, which must be
+            # taken care of in the caller by stack_frame_size_delta(-WORD)
         else:
             if IS_X86_32:
                 mc.MOV_rs(edx.value, 4 * WORD)
@@ -514,6 +525,8 @@
             assert len(set(inputargs)) == len(inputargs)
 
         self.setup(original_loop_token)
+        self.codemap.inherit_code_from_position(faildescr.adr_jump_offset)
+        self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         descr_number = compute_unique_id(faildescr)
         if log:
             operations = self._inject_debugging_code(faildescr, operations,
@@ -674,8 +687,12 @@
         self.datablockwrapper.done()      # finish using cpu.asmmemmgr
         self.datablockwrapper = None
         allblocks = self.get_asmmemmgr_blocks(looptoken)
-        return self.mc.materialize(self.cpu.asmmemmgr, allblocks,
-                                   self.cpu.gc_ll_descr.gcrootmap)
+        size = self.mc.get_relative_pos()
+        res = self.mc.materialize(self.cpu.asmmemmgr, allblocks,
+                                  self.cpu.gc_ll_descr.gcrootmap)
+        self.cpu.asmmemmgr.register_codemap(
+            self.codemap.get_final_bytecode(res, size))
+        return res
 
     def patch_jump_for_descr(self, faildescr, adr_new_target):
         adr_jump_offset = faildescr.adr_jump_offset
@@ -686,6 +703,7 @@
         # place, but clobber the recovery stub with a jump to the real
         # target.
         mc = codebuf.MachineCodeBlockWrapper()
+        mc.force_frame_size(DEFAULT_FRAME_BYTES)
         if rx86.fits_in_32bits(offset):
             mc.writeimm32(offset)
             mc.copy_to_raw_memory(adr_jump_offset)
@@ -1756,6 +1774,7 @@
 
     def generate_propagate_error_64(self):
         assert WORD == 8
+        self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         startpos = self.mc.get_relative_pos()
         self.mc.JMP(imm(self.propagate_exception_path))
         return startpos
@@ -1763,6 +1782,7 @@
     def generate_quick_failure(self, guardtok):
         """ Gather information about failure
         """
+        self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         startpos = self.mc.get_relative_pos()
         fail_descr, target = self.store_info_on_descr(startpos, guardtok)
         self.mc.PUSH(imm(fail_descr))
@@ -1838,6 +1858,9 @@
 
     def _build_failure_recovery(self, exc, withfloats=False):
         mc = codebuf.MachineCodeBlockWrapper()
+        # this is jumped to, from a stack that has DEFAULT_FRAME_BYTES
+        # followed by 2 extra words just pushed
+        mc.force_frame_size(DEFAULT_FRAME_BYTES + 2 * WORD)
         self.mc = mc
 
         self._push_all_regs_to_frame(mc, [], withfloats)
@@ -1909,6 +1932,7 @@
             self.mc.J_il(rx86.Conditions[condition], 0)
         else:
             self.mc.JMP_l(0)
+            self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         guard_token.pos_jump_offset = self.mc.get_relative_pos() - 4
         self.pending_guard_tokens.append(guard_token)
 
@@ -2020,6 +2044,7 @@
         offset = jmp_location - je_location
         assert 0 < offset <= 127
         self.mc.overwrite(je_location - 1, chr(offset))
+        self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         #
         return jmp_location
 
@@ -2104,6 +2129,8 @@
         if is_frame and align_stack:
             mc.SUB_ri(esp.value, 16 - WORD) # erase the return address
         mc.CALL(imm(self.wb_slowpath[helper_num]))
+        if not is_frame:
+            mc.stack_frame_size_delta(-WORD)
         if is_frame and align_stack:
             mc.ADD_ri(esp.value, 16 - WORD) # erase the return address
 
@@ -2340,6 +2367,7 @@
         offset = self.mc.get_relative_pos() - jmp_adr1
         assert 0 < offset <= 127
         self.mc.overwrite(jmp_adr1-1, chr(offset))
+        self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         # write down the tid, but not if it's the result of the CALL
         self.mc.MOV(mem(eax, 0), imm(arraydescr.tid))
         # while we're at it, this line is not needed if we've done the CALL
diff --git a/rpython/jit/backend/x86/callbuilder.py b/rpython/jit/backend/x86/callbuilder.py
--- a/rpython/jit/backend/x86/callbuilder.py
+++ b/rpython/jit/backend/x86/callbuilder.py
@@ -58,7 +58,7 @@
             self.fnloc_is_immediate = False
             self.fnloc = None
             self.arglocs = arglocs + [fnloc]
-        self.current_esp = 0     # 0 or (usually) negative, counted in bytes
+        self.start_frame_size = self.mc._frame_size
 
     def select_call_release_gil_mode(self):
         AbstractCallBuilder.select_call_release_gil_mode(self)
@@ -70,13 +70,15 @@
     def subtract_esp_aligned(self, count):
         if count > 0:
             align = align_stack_words(count)
-            self.current_esp -= align * WORD
             self.mc.SUB_ri(esp.value, align * WORD)
 
+    def get_current_esp(self):
+        return self.start_frame_size - self.mc._frame_size
+
     def restore_stack_pointer(self, target_esp=0):
-        if self.current_esp != target_esp:
-            self.mc.ADD_ri(esp.value, target_esp - self.current_esp)
-            self.current_esp = target_esp
+        current_esp = self.get_current_esp()
+        if current_esp != target_esp:
+            self.mc.ADD_ri(esp.value, target_esp - current_esp)
 
     def load_result(self):
         """Overridden in CallBuilder32 and CallBuilder64"""
@@ -99,9 +101,10 @@
         # after the rearrangements done just before, ignoring the return
         # value eax, if necessary
         assert not self.is_call_release_gil
-        self.change_extra_stack_depth = (self.current_esp != 0)
+        current_esp = self.get_current_esp()
+        self.change_extra_stack_depth = (current_esp != 0)
         if self.change_extra_stack_depth:
-            self.asm.set_extra_stack_depth(self.mc, -self.current_esp)
+            self.asm.set_extra_stack_depth(self.mc, -current_esp)
         noregs = self.asm.cpu.gc_ll_descr.is_shadow_stack()
         gcmap = self.asm._regalloc.get_gcmap([eax], noregs=noregs)
         self.asm.push_gcmap(self.mc, gcmap, store=True)
@@ -142,7 +145,7 @@
             # and 5/7 words as described for asmgcroot.ASM_FRAMEDATA, for a
             # total size of JIT_USE_WORDS.  This structure is found at
             # [ESP+css].
-            css = -self.current_esp + (
+            css = -self.get_current_esp() + (
                 WORD * (PASS_ON_MY_FRAME - asmgcroot.JIT_USE_WORDS))
             assert css >= 2 * WORD
             # Save ebp
@@ -178,7 +181,7 @@
             else:
                 self.tlofs_reg = r12
             self.mc.MOV_rs(self.tlofs_reg.value,
-                           THREADLOCAL_OFS - self.current_esp)
+                           THREADLOCAL_OFS - self.get_current_esp())
             if self.asm._is_asmgcc():
                 self.mc.AND_ri(self.tlofs_reg.value, ~1)
         return self.tlofs_reg
@@ -451,7 +454,10 @@
         else:
             self.mc.CALL(self.fnloc)
             if self.callconv != FFI_DEFAULT_ABI:
-                self.current_esp += self._fix_stdcall(self.callconv)
+                # in the STDCALL ABI, the CALL above has an effect on
+                # the stack depth.  Adjust 'mc._frame_size'.
+                delta = self._fix_stdcall(self.callconv)
+                self.mc.stack_frame_size_delta(-delta)
 
     def _fix_stdcall(self, callconv):
         from rpython.rlib.clibffi import FFI_STDCALL
diff --git a/rpython/jit/backend/x86/codebuf.py b/rpython/jit/backend/x86/codebuf.py
--- a/rpython/jit/backend/x86/codebuf.py
+++ b/rpython/jit/backend/x86/codebuf.py
@@ -23,6 +23,7 @@
                               codebuilder_cls):
     def __init__(self):
         self.init_block_builder()
+        codebuilder_cls.__init__(self)
         # a list of relative positions; for each position p, the bytes
         # at [p-4:p] encode an absolute address that will need to be
         # made relative.  Only works on 32-bit!
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -12,7 +12,7 @@
      valid_addressing_size)
 from rpython.jit.backend.x86 import rx86
 from rpython.jit.backend.x86.arch import (WORD, JITFRAME_FIXED_SIZE, IS_X86_32,
-    IS_X86_64)
+    IS_X86_64, DEFAULT_FRAME_BYTES)
 from rpython.jit.backend.x86.jump import remap_frame_layout_mixed
 from rpython.jit.backend.x86.regloc import (FrameLoc, RegLoc, ConstFloatLoc,
     FloatImmedLoc, ImmedLoc, imm, imm0, imm1, ecx, eax, edx, ebx, esi, edi,
@@ -314,6 +314,7 @@
         while i < len(operations):
             op = operations[i]
             self.assembler.mc.mark_op(op)
+            assert self.assembler.mc._frame_size == DEFAULT_FRAME_BYTES
             self.rm.position = i
             self.xrm.position = i
             if op.has_no_side_effect() and op.result not in self.longevity:
@@ -1323,7 +1324,7 @@
         assembler.closing_jump(self.jump_target_descr)
 
     def consider_debug_merge_point(self, op):
-        pass
+        self.assembler.debug_merge_point(op)
 
     def consider_jit_debug(self, op):
         pass
diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py
--- a/rpython/jit/backend/x86/rx86.py
+++ b/rpython/jit/backend/x86/rx86.py
@@ -454,6 +454,11 @@
 class AbstractX86CodeBuilder(object):
     """Abstract base class."""
 
+    def __init__(self):
+        self.frame_positions = []
+        self.frame_assignments = []
+        self.force_frame_size(self.WORD)
+
     def writechar(self, char):
         raise NotImplementedError
 
@@ -471,6 +476,23 @@
         self.writechar(chr((imm >> 16) & 0xFF))
         self.writechar(chr((imm >> 24) & 0xFF))
 
+    def force_frame_size(self, frame_size):
+        self.frame_positions.append(self.get_relative_pos())
+        self.frame_assignments.append(frame_size)
+        self._frame_size = frame_size
+
+    def stack_frame_size_delta(self, delta):
+        "Called when we generate an instruction that changes the value of ESP"
+        self._frame_size += delta
+        self.frame_positions.append(self.get_relative_pos()) 
+        self.frame_assignments.append(self._frame_size)
+        assert self._frame_size >= self.WORD
+
+    def check_stack_size_at_ret(self):
+        assert self._frame_size == self.WORD
+        if not we_are_translated():
+            self._frame_size = None
+
     # ------------------------------ MOV ------------------------------
 
     MOV_ri = insn(register(1), '\xB8', immediate(2))
@@ -481,14 +503,24 @@
     INC_m = insn(rex_w, '\xFF', orbyte(0), mem_reg_plus_const(1))
     INC_j = insn(rex_w, '\xFF', orbyte(0), abs_(1))
 
-    ADD_ri,ADD_rr,ADD_rb,_,_,ADD_rm,ADD_rj,_,_ = common_modes(0)
+    AD1_ri,ADD_rr,ADD_rb,_,_,ADD_rm,ADD_rj,_,_ = common_modes(0)
     OR_ri, OR_rr, OR_rb, _,_,OR_rm, OR_rj, _,_ = common_modes(1)
     AND_ri,AND_rr,AND_rb,_,_,AND_rm,AND_rj,_,_ = common_modes(4)
-    SUB_ri,SUB_rr,SUB_rb,_,_,SUB_rm,SUB_rj,SUB_ji8,SUB_mi8 = common_modes(5)
+    SU1_ri,SUB_rr,SUB_rb,_,_,SUB_rm,SUB_rj,SUB_ji8,SUB_mi8 = common_modes(5)
     SBB_ri,SBB_rr,SBB_rb,_,_,SBB_rm,SBB_rj,_,_ = common_modes(3)
     XOR_ri,XOR_rr,XOR_rb,_,_,XOR_rm,XOR_rj,_,_ = common_modes(6)
     CMP_ri,CMP_rr,CMP_rb,CMP_bi,CMP_br,CMP_rm,CMP_rj,_,_ = common_modes(7)
 
+    def ADD_ri(self, reg, immed):
+        self.AD1_ri(reg, immed)
+        if reg == R.esp:
+            self.stack_frame_size_delta(-immed)
+
+    def SUB_ri(self, reg, immed):
+        self.SU1_ri(reg, immed)
+        if reg == R.esp:
+            self.stack_frame_size_delta(+immed)
+
     CMP_mi8 = insn(rex_w, '\x83', orbyte(7<<3), mem_reg_plus_const(1), immediate(2, 'b'))
     CMP_mi32 = insn(rex_w, '\x81', orbyte(7<<3), mem_reg_plus_const(1), immediate(2))
     CMP_mi = select_8_or_32_bit_immed(CMP_mi8, CMP_mi32)
@@ -538,30 +570,65 @@
     # ------------------------------ Misc stuff ------------------------------
 
     NOP = insn('\x90')
-    RET = insn('\xC3')
-    RET16_i = insn('\xC2', immediate(1, 'h'))
+    RE1 = insn('\xC3')
+    RE116_i = insn('\xC2', immediate(1, 'h'))
 
-    PUSH_r = insn(rex_nw, register(1), '\x50')
-    PUSH_b = insn(rex_nw, '\xFF', orbyte(6<<3), stack_bp(1))
-    PUSH_m = insn(rex_nw, '\xFF', orbyte(6<<3), mem_reg_plus_const(1))
-    PUSH_i8 = insn('\x6A', immediate(1, 'b'))
-    PUSH_i32 = insn('\x68', immediate(1, 'i'))
-    def PUSH_i(mc, immed):
+    def RET(self):
+        self.check_stack_size_at_ret()
+        self.RE1()
+
+    def RET16_i(self, immed):
+        self.check_stack_size_at_ret()
+        self.RE116_i(immed)
+
+    PUS1_r = insn(rex_nw, register(1), '\x50')
+    PUS1_b = insn(rex_nw, '\xFF', orbyte(6<<3), stack_bp(1))

From noreply at buildbot.pypy.org  Thu Apr 16 10:17:52 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 16 Apr 2015 10:17:52 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge vmprof
Message-ID: <20150416081752.D6A881C071E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r76802:86cfdf3d2620
Date: 2015-04-16 10:17 +0200
http://bitbucket.org/pypy/pypy/changeset/86cfdf3d2620/

Log:	merge vmprof

diff too long, truncating to 2000 out of 2558 lines

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -9,7 +9,6 @@
 from rpython.rlib.signature import signature
 from rpython.rlib.rarithmetic import r_uint, SHRT_MIN, SHRT_MAX, \
     INT_MIN, INT_MAX, UINT_MAX, USHRT_MAX
-from rpython.rlib.rweaklist import RWeakListMixin
 
 from pypy.interpreter.executioncontext import (ExecutionContext, ActionFlag,
     UserDelAction)
@@ -367,10 +366,6 @@
 
 # ____________________________________________________________
 
-class CodeObjWeakList(RWeakListMixin):
-    def __init__(self):
-        self.initialize()
-
 class ObjSpace(object):
     """Base class for the interpreter-level implementations of object spaces.
     http://pypy.readthedocs.org/en/latest/objspace.html"""
@@ -394,7 +389,6 @@
         self.check_signal_action = None   # changed by the signal module
         self.user_del_action = UserDelAction(self)
         self._code_of_sys_exc_info = None
-        self.all_code_objs = CodeObjWeakList()
 
         # can be overridden to a subclass
         self.initialize()
@@ -672,16 +666,16 @@
             assert ec is not None
             return ec
 
+    def register_code_callback(self, callback):
+        ec = self.getexecutioncontext()
+        ec._code_callback = callback
+
     def register_code_object(self, pycode):
-        callback = self.getexecutioncontext().register_code_callback
-        if callback is not None:
-            callback(self, pycode)
-        self.all_code_objs.add_handle(pycode)
-
-    def set_code_callback(self, callback):
         ec = self.getexecutioncontext()
-        ec.register_code_callback = callback
-        
+        if ec._code_callback is None:
+            return
+        ec._code_callback(self, pycode)
+    
     def _freeze_(self):
         return True
 
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -2,6 +2,7 @@
 from pypy.interpreter.error import OperationError, get_cleared_operation_error
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib import jit
+from rpython.rlib.objectmodel import we_are_translated
 
 TICK_COUNTER_STEP = 100
 
@@ -33,11 +34,16 @@
         self.profilefunc = None
         self.w_profilefuncarg = None
         self.thread_disappeared = False   # might be set to True after os.fork()
-        self.register_code_callback = None
+
         if sys.maxint == 2147483647:
             self._code_unique_id = 0 # XXX this is wrong, it won't work on 32bit
         else:
-            self._code_unique_id = 0x7000000000000000
+            if we_are_translated():
+                self._code_unique_id = 0x7000000000000000
+            else:
+                self._code_unique_id = 0x7700000000000000
+                # should be enough code objects
+        self._code_callback = None
 
     @staticmethod
     def _mark_thread_disappeared(space):
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -129,7 +129,7 @@
 
         ec = self.space.getexecutioncontext()
         self._unique_id = ec._code_unique_id
-        ec._code_unique_id += 2 # so we have one bit that we can mark stuff
+        ec._code_unique_id += 4 # so we have two bits that we can mark stuff
         # with
 
     def _get_full_name(self):
diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -3,13 +3,15 @@
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rtyper.annlowlevel import cast_instance_to_gcref, cast_base_ptr_to_instance
 from rpython.rlib.objectmodel import we_are_translated
-from rpython.rlib import jit, rposix, entrypoint
+from rpython.rlib import jit, rposix, rgc
+from rpython.rlib.rarithmetic import ovfcheck_float_to_int
 from rpython.rtyper.tool import rffi_platform as platform
 from rpython.rlib.rstring import StringBuilder
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import oefmt, wrap_oserror, OperationError
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.pyframe import PyFrame
+from pypy.interpreter.pycode import PyCode
 
 ROOT = py.path.local(__file__).join('..')
 SRC = ROOT.join('src')
@@ -28,14 +30,13 @@
     libraries = ['unwind'],
     
     post_include_bits=["""
-        void* pypy_vmprof_get_virtual_ip(void*);
         void pypy_vmprof_init(void);
     """],
     
     separate_module_sources=["""
         void pypy_vmprof_init(void) {
             vmprof_set_mainloop(pypy_execute_frame_trampoline, 0,
-                                pypy_vmprof_get_virtual_ip);
+                                NULL);
         }
     """],
     )
@@ -56,7 +57,7 @@
 
 pypy_execute_frame_trampoline = rffi.llexternal(
     "pypy_execute_frame_trampoline",
-    [llmemory.GCREF, llmemory.GCREF, llmemory.GCREF],
+    [llmemory.GCREF, llmemory.GCREF, llmemory.GCREF, lltype.Signed],
     llmemory.GCREF,
     compilation_info=eci,
     _nowrapper=True, sandboxsafe=True,
@@ -96,23 +97,15 @@
             gc_frame = cast_instance_to_gcref(frame)
             gc_inputvalue = cast_instance_to_gcref(w_inputvalue)
             gc_operr = cast_instance_to_gcref(operr)
-            gc_result = pypy_execute_frame_trampoline(gc_frame, gc_inputvalue, gc_operr)
+            assert frame.pycode._unique_id & 3 == 0
+            unique_id = frame.pycode._unique_id | 1
+            gc_result = pypy_execute_frame_trampoline(gc_frame, gc_inputvalue,
+                                                      gc_operr, unique_id)
             return cast_base_ptr_to_instance(W_Root, gc_result)
         else:
             return original_execute_frame(frame, w_inputvalue, operr)
 
 
- at entrypoint.entrypoint_lowlevel('main', [llmemory.GCREF],
-                                'pypy_vmprof_get_virtual_ip', True)
-def get_virtual_ip(gc_frame):
-    frame = cast_base_ptr_to_instance(PyFrame, gc_frame)
-    if jit._get_virtualizable_token(frame):
-        return rffi.cast(rffi.VOIDP, 0)
-    virtual_ip = do_get_virtual_ip(frame)
-    return rffi.cast(rffi.VOIDP, virtual_ip)
-
-def do_get_virtual_ip(frame):
-    return frame.pycode._unique_id
 
 def write_long_to_string_builder(l, b):
     if sys.maxint == 2147483647:
@@ -130,31 +123,33 @@
         b.append(chr((l >> 48) & 0xff))
         b.append(chr((l >> 56) & 0xff))
 
+def try_cast_to_pycode(gcref):
+    return rgc.try_cast_gcref_to_instance(PyCode, gcref)
+
+MAX_CODES = 1000
+
 class VMProf(object):
     def __init__(self):
         self.is_enabled = False
         self.ever_enabled = False
-        self.mapping_so_far = [] # stored mapping in between runs
         self.fileno = -1
+        self.current_codes = []
 
-    def enable(self, space, fileno, period):
+    def enable(self, space, fileno, period_usec):
         if self.is_enabled:
             raise oefmt(space.w_ValueError, "_vmprof already enabled")
         self.fileno = fileno
         self.is_enabled = True
-        self.write_header(fileno, period)
+        self.write_header(fileno, period_usec)
         if not self.ever_enabled:
             if we_are_translated():
                 pypy_vmprof_init()
             self.ever_enabled = True
-        for weakcode in space.all_code_objs.get_all_handles():
-            code = weakcode()
-            if code:
-                self.register_code(space, code)
-        space.set_code_callback(vmprof_register_code)
+        self.gather_all_code_objs(space)
+        space.register_code_callback(vmprof_register_code)
         if we_are_translated():
             # does not work untranslated
-            res = vmprof_enable(fileno, period, 0,
+            res = vmprof_enable(fileno, period_usec, 0,
                                 lltype.nullptr(rffi.CCHARP.TO), 0)
         else:
             res = 0
@@ -162,42 +157,55 @@
             raise wrap_oserror(space, OSError(rposix.get_saved_errno(),
                                               "_vmprof.enable"))
 
-    def write_header(self, fileno, period):
-        if period == -1:
-            period_usec = 1000000 / 100 #  100hz
-        else:
-            period_usec = period
+    def gather_all_code_objs(self, space):
+        all_code_objs = rgc.do_get_objects(try_cast_to_pycode)
+        for code in all_code_objs:
+            self.register_code(space, code)
+
+    def write_header(self, fileno, period_usec):
+        assert period_usec > 0
         b = StringBuilder()
         write_long_to_string_builder(0, b)
         write_long_to_string_builder(3, b)
         write_long_to_string_builder(0, b)
         write_long_to_string_builder(period_usec, b)
         write_long_to_string_builder(0, b)
+        b.append('\x04') # interp name
+        b.append(chr(len('pypy')))
+        b.append('pypy')
         os.write(fileno, b.build())
 
     def register_code(self, space, code):
         if self.fileno == -1:
             raise OperationError(space.w_RuntimeError,
                                  space.wrap("vmprof not running"))
-        name = code._get_full_name()
+        self.current_codes.append(code)
+        if len(self.current_codes) >= MAX_CODES:
+            self._flush_codes(space)
+
+    def _flush_codes(self, space):
         b = StringBuilder()
-        b.append('\x02')
-        write_long_to_string_builder(code._unique_id, b)
-        write_long_to_string_builder(len(name), b)
-        b.append(name)
+        for code in self.current_codes:
+            name = code._get_full_name()
+            b.append('\x02')
+            write_long_to_string_builder(code._unique_id, b)
+            write_long_to_string_builder(len(name), b)
+            b.append(name)
         os.write(self.fileno, b.build())
+        self.current_codes = []
 
     def disable(self, space):
         if not self.is_enabled:
             raise oefmt(space.w_ValueError, "_vmprof not enabled")
         self.is_enabled = False
+        space.register_code_callback(None)
+        self._flush_codes(space)
         self.fileno = -1
         if we_are_translated():
            # does not work untranslated
             res = vmprof_disable()
         else:
             res = 0
-        space.set_code_callback(None)
         if res == -1:
             raise wrap_oserror(space, OSError(rposix.get_saved_errno(),
                                               "_vmprof.disable"))
@@ -207,13 +215,23 @@
     mod_vmprof = space.getbuiltinmodule('_vmprof')
     assert isinstance(mod_vmprof, Module)
     mod_vmprof.vmprof.register_code(space, code)
-        
- at unwrap_spec(fileno=int, period=int)
-def enable(space, fileno, period=-1):
+
+ at unwrap_spec(fileno=int, period=float)
+def enable(space, fileno, period=0.01):   # default 100 Hz
     from pypy.module._vmprof import Module
     mod_vmprof = space.getbuiltinmodule('_vmprof')
     assert isinstance(mod_vmprof, Module)
-    mod_vmprof.vmprof.enable(space, fileno, period)
+    #
+    try:
+        period_usec = ovfcheck_float_to_int(period * 1000000.0 + 0.5)
+        if period_usec <= 0 or period_usec >= 1e6:
+            # we don't want seconds here at all
+            raise ValueError
+    except (ValueError, OverflowError):
+        raise OperationError(space.w_ValueError,
+                             space.wrap("'period' too large or non positive"))
+    #
+    mod_vmprof.vmprof.enable(space, fileno, period_usec)
 
 def disable(space):
     from pypy.module._vmprof import Module
diff --git a/pypy/module/_vmprof/src/fake_pypy_api.c b/pypy/module/_vmprof/src/fake_pypy_api.c
--- a/pypy/module/_vmprof/src/fake_pypy_api.c
+++ b/pypy/module/_vmprof/src/fake_pypy_api.c
@@ -1,25 +1,15 @@
-
-long pypy_jit_start_addr(void)
-{
-	return 3;
-}
-
-long pypy_jit_end_addr(void)
-{
-	return 3;
-}
 
 long pypy_jit_stack_depth_at_loc(long x)
 {
 	return 0;
 }
 
-long pypy_find_codemap_at_addr(long x)
+void *pypy_find_codemap_at_addr(long x)
 {
-	return 0;
+	return (void *)0;
 }
 
-long pypy_yield_codemap_at_addr(long x, long y, long *a)
+long pypy_yield_codemap_at_addr(void *x, long y, long *a)
 {
 	return 0;
 }
@@ -27,3 +17,5 @@
 void pypy_pyframe_execute_frame(void)
 {
 }
+
+volatile int pypy_codemap_currently_invalid = 0;
diff --git a/pypy/module/_vmprof/src/get_custom_offset.c b/pypy/module/_vmprof/src/get_custom_offset.c
--- a/pypy/module/_vmprof/src/get_custom_offset.c
+++ b/pypy/module/_vmprof/src/get_custom_offset.c
@@ -1,46 +1,65 @@
 
-long pypy_jit_start_addr();
-long pypy_jit_end_addr();
-long pypy_jit_stack_depth_at_loc(long);
-long pypy_find_codemap_at_addr(long);
-long pypy_yield_codemap_at_addr(long, long, long*);
+extern volatile int pypy_codemap_currently_invalid;
+
+void *pypy_find_codemap_at_addr(long addr, long *start_addr);
+long pypy_yield_codemap_at_addr(void *codemap_raw, long addr,
+                                long *current_pos_addr);
+long pypy_jit_stack_depth_at_loc(long loc);
+
 
 void vmprof_set_tramp_range(void* start, void* end)
 {
 }
 
-static ptrdiff_t vmprof_unw_get_custom_offset(void* ip, unw_cursor_t *cp) {
-	intptr_t ip_l = (intptr_t)ip;
+int custom_sanity_check()
+{
+    return !pypy_codemap_currently_invalid;
+}
 
-	if (ip_l < pypy_jit_start_addr() || ip_l > pypy_jit_end_addr()) {
-		return -1;
-	}
-	return (void*)pypy_jit_stack_depth_at_loc(ip_l);
+static ptrdiff_t vmprof_unw_get_custom_offset(void* ip, void *cp) {
+    intptr_t ip_l = (intptr_t)ip;
+    return pypy_jit_stack_depth_at_loc(ip_l);
 }
 
 static long vmprof_write_header_for_jit_addr(void **result, long n,
-											 void *ip, int max_depth)
+                                             void *ip, int max_depth)
 {
-	long codemap_pos;
-	long current_pos = 0;
-	intptr_t id;
-	intptr_t addr = (intptr_t)ip;
+    void *codemap;
+    long current_pos = 0;
+    intptr_t id;
+    long start_addr = 0;
+    intptr_t addr = (intptr_t)ip;
+    int start, k;
+    void *tmp;
 
-	if (addr < pypy_jit_start_addr() || addr > pypy_jit_end_addr()) {
-		return n;
-	}
-	codemap_pos = pypy_find_codemap_at_addr(addr);
-	if (codemap_pos == -1) {
-		return n;
-	}
-	while (1) {
-		id = pypy_yield_codemap_at_addr(codemap_pos, addr, ¤t_pos);
-		if (id == 0) {
-			return n;
-		}
-		result[n++] = id;
-		if (n >= max_depth) {
-			return n;
-		}
-	}
+    codemap = pypy_find_codemap_at_addr(addr, &start_addr);
+    if (codemap == NULL)
+        // not a jit code at all
+        return n;
+
+    // modify the last entry to point to start address and not the random one
+    // in the middle
+    result[n - 1] = (void*)start_addr;
+    start = n;
+    while (n < max_depth) {
+        id = pypy_yield_codemap_at_addr(codemap, addr, ¤t_pos);
+        if (id == 0)
+            // finish
+            break;
+        result[n++] = (void *)id;
+    }
+    // we strip the topmost part - the reason is that it's either
+    // represented in the jitted caller or it's not jitted (we have the
+    // same function essentially twice
+    k = 0;
+    while (k < (n - start) / 2) {
+        tmp = result[start + k];
+        result[start + k] = result[n - k - 1];
+        result[n - k - 1] = tmp;
+        k++;
+    }
+    if (n != max_depth) {
+        n--;
+    }
+    return n;
 }
diff --git a/pypy/module/_vmprof/src/trampoline.asmgcc.s b/pypy/module/_vmprof/src/trampoline.asmgcc.s
--- a/pypy/module/_vmprof/src/trampoline.asmgcc.s
+++ b/pypy/module/_vmprof/src/trampoline.asmgcc.s
@@ -6,11 +6,10 @@
 	.type	pypy_execute_frame_trampoline, @function
 pypy_execute_frame_trampoline:
 	.cfi_startproc
-	pushq	%rdi
+	pushq	%rcx
 	.cfi_def_cfa_offset 16
 	call pypy_pyframe_execute_frame at PLT
-	/* GCROOT 0(%rsp) */
-	popq	%rdi
+	popq	%rcx
 	.cfi_def_cfa_offset 8
 	ret
 	.cfi_endproc
diff --git a/pypy/module/_vmprof/src/trampoline.h b/pypy/module/_vmprof/src/trampoline.h
--- a/pypy/module/_vmprof/src/trampoline.h
+++ b/pypy/module/_vmprof/src/trampoline.h
@@ -1,1 +1,1 @@
-void* pypy_execute_frame_trampoline(void*, void*, void*);
+void* pypy_execute_frame_trampoline(void*, void*, void*, long);
diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c
--- a/pypy/module/_vmprof/src/vmprof.c
+++ b/pypy/module/_vmprof/src/vmprof.c
@@ -25,6 +25,8 @@
 #include 
 #include 
 #include 
+#include 
+#include 
 
 #define UNW_LOCAL_ONLY
 #include 
@@ -34,12 +36,18 @@
 #define _unused(x) ((void)x)
 
 #define MAX_FUNC_NAME 128
-#define MAX_STACK_DEPTH 64
+#define MAX_STACK_DEPTH 1024
+#define BUFFER_SIZE 8192
 
-static FILE* profile_file = NULL;
+
+static int profile_file = 0;
+static char profile_write_buffer[BUFFER_SIZE];
+static int profile_buffer_position = 0;
 void* vmprof_mainloop_func;
 static ptrdiff_t mainloop_sp_offset;
 static vmprof_get_virtual_ip_t mainloop_get_virtual_ip;
+static long last_period_usec = 0;
+static int atfork_hook_installed = 0;
 
 
 /* *************************************************************
@@ -51,27 +59,33 @@
 #define MARKER_VIRTUAL_IP '\x02'
 #define MARKER_TRAILER '\x03'
 
-static void prof_word(FILE* f, long x) {
-    fwrite(&x, sizeof(x), 1, f);
+static void prof_word(long x) {
+	((long*)(profile_write_buffer + profile_buffer_position))[0] = x;
+	profile_buffer_position += sizeof(long);
 }
 
-static void prof_header(FILE* f, long period_usec) {
-    prof_word(f, 0);
-    prof_word(f, 3);
-    prof_word(f, 0);
-    prof_word(f, period_usec);
-    prof_word(f, 0);
+static void prof_header(long period_usec) {
+    // XXX never used here?
+    prof_word(0);
+    prof_word(3);
+    prof_word(0);
+    prof_word(period_usec);
+    prof_word(0);
+    write(profile_file, profile_write_buffer, profile_buffer_position);
+    profile_buffer_position = 0;
 }
 
-static void prof_write_stacktrace(FILE* f, void** stack, int depth, int count) {
+static void prof_write_stacktrace(void** stack, int depth, int count) {
     int i;
 	char marker = MARKER_STACKTRACE;
 
-	fwrite(&marker, 1, 1, f);
-    prof_word(f, count);
-    prof_word(f, depth);
+	profile_write_buffer[profile_buffer_position++] = MARKER_STACKTRACE;
+    prof_word(count);
+    prof_word(depth);
     for(i=0; i= 0);
     _unused(ret);
+	int first_run = 1;
 
     while (n < max_depth) {
         if (unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *) &ip) < 0) {
@@ -173,16 +192,21 @@
           void **arg_ptr = (void**)arg_addr;
           // fprintf(stderr, "stacktrace mainloop: rsp %p   &f2 %p   offset %ld\n", 
           //         sp, arg_addr, mainloop_sp_offset);
-          ip = mainloop_get_virtual_ip(*arg_ptr);
+		  if (mainloop_get_virtual_ip) {
+			  ip = mainloop_get_virtual_ip(*arg_ptr);
+		  } else {
+			  ip = *arg_ptr;
+		  }
         }
 
         result[n++] = ip;
 		n = vmprof_write_header_for_jit_addr(result, n, ip, max_depth);
-        if (vmprof_unw_step(&cursor) <= 0) {
+        if (vmprof_unw_step(&cursor, first_run) <= 0) {
             break;
         }
+		first_run = 0;
     }
-    --recursive;
+    //--recursive;
     return n;
 }
 
@@ -193,10 +217,12 @@
 
 static void sigprof_handler(int sig_nr, siginfo_t* info, void *ucontext) {
     void* stack[MAX_STACK_DEPTH];
+    int saved_errno = errno;
     stack[0] = GetPC((ucontext_t*)ucontext);
     int depth = frame_forcer(get_stack_trace(stack+1, MAX_STACK_DEPTH-1, ucontext));
     depth++;  // To account for pc value in stack[0];
-    prof_write_stacktrace(profile_file, stack, depth, 1);
+    prof_write_stacktrace(stack, depth, 1);
+    errno = saved_errno;
 }
 
 /* *************************************************************
@@ -209,14 +235,12 @@
 	if ((fd = dup(fd)) == -1) {
 		return -1;
 	}
-    profile_file = fdopen(fd, "wb");
-	if (!profile_file) {
-		return -1;
-	}
+	profile_buffer_position = 0;
+    profile_file = fd;
 	if (write_header)
-		prof_header(profile_file, period_usec);
+		prof_header(period_usec);
 	if (s)
-		fwrite(s, slen, 1, profile_file);
+		write(profile_file, s, slen);
 	return 0;
 }
 
@@ -226,16 +250,16 @@
     char buf[BUFSIZ];
     size_t size;
 	int marker = MARKER_TRAILER;
-	fwrite(&marker, 1, 1, profile_file);
+	write(profile_file, &marker, 1);
 
     // copy /proc/PID/maps to the end of the profile file
     sprintf(buf, "/proc/%d/maps", getpid());
     src = fopen(buf, "r");    
     while ((size = fread(buf, 1, BUFSIZ, src))) {
-        fwrite(buf, 1, size, profile_file);
+        write(profile_file, buf, size);
     }
     fclose(src);
-    fclose(profile_file);
+    close(profile_file);
 	return 0;
 }
 
@@ -253,15 +277,16 @@
 }
 
 static int remove_sigprof_handler(void) {
-    //sighandler_t res = signal(SIGPROF, SIG_DFL);
-	//if (res == SIG_ERR) {
-	//	return -1;
-	//}
+    sighandler_t res = signal(SIGPROF, SIG_DFL);
+	if (res == SIG_ERR) {
+		return -1;
+	}
 	return 0;
 };
 
 static int install_sigprof_timer(long period_usec) {
     static struct itimerval timer;
+    last_period_usec = period_usec;
     timer.it_interval.tv_sec = 0;
     timer.it_interval.tv_usec = period_usec;
     timer.it_value = timer.it_interval;
@@ -273,15 +298,45 @@
 
 static int remove_sigprof_timer(void) {
     static struct itimerval timer;
+    last_period_usec = 0;
     timer.it_interval.tv_sec = 0;
     timer.it_interval.tv_usec = 0;
-    timer.it_value = timer.it_interval;
+    timer.it_value.tv_sec = 0;
+    timer.it_value.tv_usec = 0;
     if (setitimer(ITIMER_PROF, &timer, NULL) != 0) {
 		return -1;
     }
 	return 0;
 }
 
+static void atfork_disable_timer(void) {
+    remove_sigprof_timer();
+}
+
+static void atfork_enable_timer(void) {
+    install_sigprof_timer(last_period_usec);
+}
+
+static int install_pthread_atfork_hooks(void) {
+    /* this is needed to prevent the problems described there:
+         - http://code.google.com/p/gperftools/issues/detail?id=278
+         - http://lists.debian.org/debian-glibc/2010/03/msg00161.html
+
+        TL;DR: if the RSS of the process is large enough, the clone() syscall
+        will be interrupted by the SIGPROF before it can complete, then
+        retried, interrupted again and so on, in an endless loop.  The
+        solution is to disable the timer around the fork, and re-enable it
+        only inside the parent.
+    */
+    if (atfork_hook_installed)
+        return 0;
+    int ret = pthread_atfork(atfork_disable_timer, atfork_enable_timer, NULL);
+    if (ret != 0)
+        return -1;
+    atfork_hook_installed = 1;
+    return 0;
+}
+
 /* *************************************************************
  * public API
  * *************************************************************
@@ -297,8 +352,7 @@
 int vmprof_enable(int fd, long period_usec, int write_header, char *s,
 				  int slen)
 {
-    if (period_usec == -1)
-        period_usec = 1000000 / 100; /* 100hz */
+    assert(period_usec > 0);
     if (open_profile(fd, period_usec, write_header, s, slen) == -1) {
 		return -1;
 	}
@@ -308,6 +362,9 @@
     if (install_sigprof_timer(period_usec) == -1) {
 		return -1;
 	}
+    if (install_pthread_atfork_hooks() == -1) {
+        return -1;
+    }
 	return 0;
 }
 
@@ -325,6 +382,7 @@
 }
 
 void vmprof_register_virtual_function(const char* name, void* start, void* end) {
+	// XXX unused by pypy
     // for now *end is simply ignored
 	char buf[1024];
 	int lgt = strlen(name) + 2 * sizeof(long) + 1;
@@ -336,5 +394,5 @@
 	((void **)(((void*)buf) + 1))[0] = start;
 	((long *)(((void*)buf) + 1 + sizeof(long)))[0] = lgt - 2 * sizeof(long) - 1;
 	strncpy(buf + 2 * sizeof(long) + 1, name, 1024 - 2 * sizeof(long) - 1);
-	fwrite(buf, lgt, 1, profile_file);
+	write(profile_file, buf, lgt);
 }
diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py
--- a/pypy/module/_vmprof/test/test__vmprof.py
+++ b/pypy/module/_vmprof/test/test__vmprof.py
@@ -21,6 +21,11 @@
             i = 0
             count = 0
             i += 5 * WORD # header
+            assert s[i] == '\x04'
+            i += 1 # marker
+            assert s[i] == '\x04'
+            i += 1 # length
+            i += len('pypy')
             while i < len(s):
                 assert s[i] == '\x02'
                 i += 1
@@ -53,3 +58,11 @@
         assert "py:foo:" in s
         assert "py:foo2:" in s
         assert no_of_codes2 >= no_of_codes + 2 # some extra codes from tests
+
+    def test_enable_ovf(self):
+        import _vmprof
+        raises(ValueError, _vmprof.enable, 999, 0)
+        raises(ValueError, _vmprof.enable, 999, -2.5)
+        raises(ValueError, _vmprof.enable, 999, 1e300)
+        raises(ValueError, _vmprof.enable, 999, 1e300 * 1e300)
+        raises(ValueError, _vmprof.enable, 999, (1e300*1e300) / (1e300*1e300))
diff --git a/pypy/module/_vmprof/test/test_direct.py b/pypy/module/_vmprof/test/test_direct.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/test/test_direct.py
@@ -0,0 +1,66 @@
+
+import cffi, py
+
+srcdir = py.path.local(__file__).join("..", "..", "src")
+
+ffi = cffi.FFI()
+ffi.cdef("""
+long vmprof_write_header_for_jit_addr(void **, long, void*, int);
+void *pypy_find_codemap_at_addr(long addr, long *start_addr);
+long pypy_yield_codemap_at_addr(void *codemap_raw, long addr,
+                                long *current_pos_addr);
+long buffer[];
+""")
+
+lib = ffi.verify("""
+volatile int pypy_codemap_currently_invalid = 0;
+
+long buffer[] = {0, 0, 0, 0, 0};
+
+
+
+void *pypy_find_codemap_at_addr(long addr, long *start_addr)
+{
+    return (void*)buffer;
+}
+
+long pypy_yield_codemap_at_addr(void *codemap_raw, long addr,
+                                long *current_pos_addr)
+{
+    long c = *current_pos_addr;
+    if (c >= 5)
+        return 0;
+    *current_pos_addr = c + 1;
+    return *((long*)codemap_raw + c);
+}
+
+
+""" + open(str(srcdir.join("get_custom_offset.c"))).read())
+
+class TestDirect(object):
+    def test_infrastructure(self):
+        cont = ffi.new("long[1]", [0])
+        buf = lib.pypy_find_codemap_at_addr(0, cont)
+        assert buf
+        cont[0] = 0
+        next_addr = lib.pypy_yield_codemap_at_addr(buf, 0, cont)
+        assert cont[0] == 1
+        assert not next_addr
+        lib.buffer[0] = 13
+        cont[0] = 0
+        next_addr = lib.pypy_yield_codemap_at_addr(buf, 0, cont)
+        assert int(ffi.cast("long", next_addr)) == 13
+
+    def test_write_header_for_jit_addr(self):
+        lib.buffer[0] = 4
+        lib.buffer[1] = 8
+        lib.buffer[2] = 12
+        lib.buffer[3] = 16
+        lib.buffer[4] = 0
+        buf = ffi.new("long[5]", [0] * 5)
+        result = ffi.cast("void**", buf)
+        res = lib.vmprof_write_header_for_jit_addr(result, 0, ffi.NULL, 100)
+        assert res == 3
+        assert buf[0] == 16
+        assert buf[1] == 12
+        assert buf[2] == 8
diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py
--- a/pypy/module/gc/referents.py
+++ b/pypy/module/gc/referents.py
@@ -44,30 +44,6 @@
     return OperationError(space.w_NotImplementedError,
                           space.wrap("operation not implemented by this GC"))
 
-# ____________________________________________________________
-
-def clear_gcflag_extra(fromlist):
-    pending = fromlist[:]
-    while pending:
-        gcref = pending.pop()
-        if rgc.get_gcflag_extra(gcref):
-            rgc.toggle_gcflag_extra(gcref)
-            pending.extend(rgc.get_rpy_referents(gcref))
-
-def do_get_objects():
-    roots = [gcref for gcref in rgc.get_rpy_roots() if gcref]
-    pending = roots[:]
-    result_w = []
-    while pending:
-        gcref = pending.pop()
-        if not rgc.get_gcflag_extra(gcref):
-            rgc.toggle_gcflag_extra(gcref)
-            w_obj = try_cast_gcref_to_w_root(gcref)
-            if w_obj is not None:
-                result_w.append(w_obj)
-            pending.extend(rgc.get_rpy_referents(gcref))
-    clear_gcflag_extra(roots)
-    return result_w
 
 # ____________________________________________________________
 
@@ -116,8 +92,8 @@
                 break
     # done.  Clear flags carefully
     rgc.toggle_gcflag_extra(gcarg)
-    clear_gcflag_extra(roots)
-    clear_gcflag_extra([gcarg])
+    rgc.clear_gcflag_extra(roots)
+    rgc.clear_gcflag_extra([gcarg])
     return result_w
 
 # ____________________________________________________________
@@ -189,8 +165,7 @@
     """Return a list of all app-level objects."""
     if not rgc.has_gcflag_extra():
         raise missing_operation(space)
-    result_w = do_get_objects()
-    rgc.assert_no_more_gcflags()
+    result_w = rgc.do_get_objects(try_cast_gcref_to_w_root)
     return space.newlist(result_w)
 
 def get_referents(space, args_w):
diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py
--- a/pypy/module/pypyjit/interp_resop.py
+++ b/pypy/module/pypyjit/interp_resop.py
@@ -105,7 +105,7 @@
             ofs = ops_offset.get(op, 0)
         if op.opnum == rop.DEBUG_MERGE_POINT:
             jd_sd = jitdrivers_sd[op.getarg(0).getint()]
-            greenkey = op.getarglist()[3:]
+            greenkey = op.getarglist()[4:]
             repr = jd_sd.warmstate.get_location_str(greenkey)
             w_greenkey = wrap_greenkey(space, jd_sd.jitdriver, greenkey, repr)
             l_w.append(DebugMergePoint(space, jit_hooks._cast_to_gcref(op),
diff --git a/pypy/module/pypyjit/test/test_jit_hook.py b/pypy/module/pypyjit/test/test_jit_hook.py
--- a/pypy/module/pypyjit/test/test_jit_hook.py
+++ b/pypy/module/pypyjit/test/test_jit_hook.py
@@ -55,7 +55,7 @@
         oplist = parse("""
         [i1, i2, p2]
         i3 = int_add(i1, i2)
-        debug_merge_point(0, 0, 0, 0, 0, ConstPtr(ptr0))
+        debug_merge_point(0, 0, 0, 0, 0, 0, ConstPtr(ptr0))
         guard_nonnull(p2) []
         guard_true(i3) []
         """, namespace={'ptr0': code_gcref}).operations
diff --git a/rpython/bin/rpython-vmprof b/rpython/bin/rpython-vmprof
new file mode 100755
--- /dev/null
+++ b/rpython/bin/rpython-vmprof
@@ -0,0 +1,28 @@
+#!/usr/bin/env pypy
+
+"""RPython translation usage:
+
+rpython  target 
+
+run with --help for more information
+"""
+
+import sys, os
+sys.path.insert(0, os.path.dirname(os.path.dirname(
+                       os.path.dirname(os.path.realpath(__file__)))))
+from rpython.translator.goal.translate import main
+
+# no implicit targets
+if len(sys.argv) == 1:
+    print __doc__
+    sys.exit(1)
+
+import _vmprof, subprocess
+x = subprocess.Popen('gzip > vmprof.log.gz', shell=True, stdin=subprocess.PIPE)
+_vmprof.enable(x.stdin.fileno(), 0.001)
+try:
+    main()
+finally:
+    _vmprof.disable()
+    x.stdin.close()
+    x.wait()
diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -102,7 +102,7 @@
         self.store_reg(mc, r.r0, r.fp, ofs)
         mc.MOV_rr(r.r0.value, r.fp.value)
         self.gen_func_epilog(mc)
-        rawstart = mc.materialize(self.cpu.asmmemmgr, [])
+        rawstart = mc.materialize(self.cpu, [])
         self.propagate_exception_path = rawstart
 
     def _store_and_reset_exception(self, mc, excvalloc=None, exctploc=None,
@@ -198,7 +198,7 @@
         mc.ADD_ri(r.sp.value, r.sp.value, (len(r.argument_regs) + 2) * WORD)
         mc.B(self.propagate_exception_path)
         #
-        rawstart = mc.materialize(self.cpu.asmmemmgr, [])
+        rawstart = mc.materialize(self.cpu, [])
         self.stack_check_slowpath = rawstart
 
     def _build_wb_slowpath(self, withcards, withfloats=False, for_frame=False):
@@ -255,7 +255,7 @@
         #
         mc.POP([r.ip.value, r.pc.value])
         #
-        rawstart = mc.materialize(self.cpu.asmmemmgr, [])
+        rawstart = mc.materialize(self.cpu, [])
         if for_frame:
             self.wb_slowpath[4] = rawstart
         else:
@@ -276,7 +276,7 @@
                                       callee_only)
         # return
         mc.POP([r.ip.value, r.pc.value])
-        return mc.materialize(self.cpu.asmmemmgr, [])
+        return mc.materialize(self.cpu, [])
 
     def _build_malloc_slowpath(self, kind):
         """ While arriving on slowpath, we have a gcpattern on stack 0.
@@ -352,7 +352,7 @@
         mc.POP([r.ip.value, r.pc.value])
 
         #
-        rawstart = mc.materialize(self.cpu.asmmemmgr, [])
+        rawstart = mc.materialize(self.cpu, [])
         return rawstart
 
     def _reload_frame_if_necessary(self, mc):
@@ -473,7 +473,7 @@
         mc.MOV_rr(r.r0.value, r.fp.value)
         #
         self.gen_func_epilog(mc)
-        rawstart = mc.materialize(self.cpu.asmmemmgr, [])
+        rawstart = mc.materialize(self.cpu, [])
         self.failure_recovery_code[exc + 2 * withfloats] = rawstart
 
     def generate_quick_failure(self, guardtok):
@@ -851,7 +851,7 @@
         # restore registers
         self._pop_all_regs_from_jitframe(mc, [], self.cpu.supports_floats)
         mc.POP([r.ip.value, r.pc.value])  # return
-        self._frame_realloc_slowpath = mc.materialize(self.cpu.asmmemmgr, [])
+        self._frame_realloc_slowpath = mc.materialize(self.cpu, [])
 
     def _load_shadowstack_top(self, mc, reg, gcrootmap):
         rst = gcrootmap.get_root_stack_top_addr()
@@ -881,7 +881,7 @@
         self.datablockwrapper = None
         allblocks = self.get_asmmemmgr_blocks(looptoken)
         size = self.mc.get_relative_pos() 
-        res = self.mc.materialize(self.cpu.asmmemmgr, allblocks,
+        res = self.mc.materialize(self.cpu, allblocks,
                                    self.cpu.gc_ll_descr.gcrootmap)
         self.cpu.asmmemmgr.register_codemap(
             self.codemap.get_final_bytecode(res, size))
diff --git a/rpython/jit/backend/arm/runner.py b/rpython/jit/backend/arm/runner.py
--- a/rpython/jit/backend/arm/runner.py
+++ b/rpython/jit/backend/arm/runner.py
@@ -50,6 +50,7 @@
     def setup_once(self):
         self.cpuinfo.arch_version = detect_arch_version()
         self.cpuinfo.hf_abi = detect_hardfloat()
+        self.codemap.setup()
         self.assembler.setup_once()
 
     def finish_once(self):
diff --git a/rpython/jit/backend/arm/test/support.py b/rpython/jit/backend/arm/test/support.py
--- a/rpython/jit/backend/arm/test/support.py
+++ b/rpython/jit/backend/arm/test/support.py
@@ -24,7 +24,7 @@
 
 def run_asm(asm):
     BOOTSTRAP_TP = lltype.FuncType([], lltype.Signed)
-    addr = asm.mc.materialize(asm.cpu.asmmemmgr, [], None)
+    addr = asm.mc.materialize(asm.cpu, [], None)
     assert addr % 8 == 0
     func = rffi.cast(lltype.Ptr(BOOTSTRAP_TP), addr)
     asm.mc._dump_trace(addr, 'test.asm')
diff --git a/rpython/jit/backend/arm/test/test_calling_convention.py b/rpython/jit/backend/arm/test/test_calling_convention.py
--- a/rpython/jit/backend/arm/test/test_calling_convention.py
+++ b/rpython/jit/backend/arm/test/test_calling_convention.py
@@ -29,7 +29,7 @@
         mc = InstrBuilder()
         mc.MOV_rr(r.r0.value, r.sp.value)
         mc.MOV_rr(r.pc.value, r.lr.value)
-        return mc.materialize(self.cpu.asmmemmgr, [])
+        return mc.materialize(self.cpu, [])
 
     def get_alignment_requirements(self):
         return 8
diff --git a/rpython/jit/backend/llsupport/asmmemmgr.py b/rpython/jit/backend/llsupport/asmmemmgr.py
--- a/rpython/jit/backend/llsupport/asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/asmmemmgr.py
@@ -5,9 +5,6 @@
 from rpython.rlib.debug import debug_start, debug_print, debug_stop
 from rpython.rlib.debug import have_debug_prints
 from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rlib.rbisect import bisect, bisect_tuple
-
-_memmngr = None # global reference so we can use @entrypoint :/
 
 
 class AsmMemoryManager(object):
@@ -27,12 +24,6 @@
         self.free_blocks = {}      # map {start: stop}
         self.free_blocks_end = {}  # map {stop: start}
         self.blocks_by_size = [[] for i in range(self.num_indices)]
-        # two lists of jit addresses (sorted) and the corresponding stack
-        # depths
-        self.jit_addr_map = []
-        self.jit_frame_depth_map = []
-        self.jit_codemap = []
-        # see codemap.py
 
     def malloc(self, minsize, maxsize):
         """Allocate executable memory, between minsize and maxsize bytes,
@@ -54,13 +45,6 @@
         if r_uint is not None:
             self.total_mallocs -= r_uint(stop - start)
         self._add_free_block(start, stop)
-        # fix up jit_addr_map
-        jit_adr_start = bisect(self.jit_addr_map, start)
-        jit_adr_stop = bisect(self.jit_addr_map, stop)
-        self.jit_addr_map = (self.jit_addr_map[:jit_adr_start] +
-                             self.jit_addr_map[jit_adr_stop:])
-        self.jit_frame_depth_map = (self.jit_frame_depth_map[:jit_adr_start] +
-                                    self.jit_frame_depth_map[jit_adr_stop:])
 
     def open_malloc(self, minsize):
         """Allocate at least minsize bytes.  Returns (start, stop)."""
@@ -167,35 +151,6 @@
         del self.free_blocks_end[stop]
         return (start, stop)
 
-    def register_frame_depth_map(self, rawstart, frame_positions,
-                                 frame_assignments):
-        if not frame_positions:
-            return
-        if not self.jit_addr_map or rawstart > self.jit_addr_map[-1]:
-            start = len(self.jit_addr_map)
-            self.jit_addr_map += [0] * len(frame_positions)
-            self.jit_frame_depth_map += [0] * len(frame_positions)
-        else:
-            start = bisect(self.jit_addr_map, rawstart)
-            self.jit_addr_map = (self.jit_addr_map[:start] +
-                                 [0] * len(frame_positions) +
-                                 self.jit_addr_map[start:])
-            self.jit_frame_depth_map = (self.jit_frame_depth_map[:start] +
-                                 [0] * len(frame_positions) +
-                                 self.jit_frame_depth_map[start:])
-        for i, pos in enumerate(frame_positions):
-            self.jit_addr_map[i + start] = pos + rawstart
-            self.jit_frame_depth_map[i + start] = frame_assignments[i]
-
-    def register_codemap(self, codemap):
-        start = codemap[0]
-        pos = bisect_tuple(self.jit_codemap, start)
-        if pos == len(self.jit_codemap): # common case
-            self.jit_codemap.append(codemap)
-        else:
-            self.jit_codemap = (self.jit_codemap[:pos] + [codemap] +
-                                self.jit_codemap[pos:])
-
     def _delete(self):
         "NOT_RPYTHON"
         if self._allocated:
@@ -351,11 +306,11 @@
             #
         debug_stop(logname)
 
-    def materialize(self, asmmemmgr, allblocks, gcrootmap=None):
+    def materialize(self, cpu, allblocks, gcrootmap=None):
         size = self.get_relative_pos()
         align = self.ALIGN_MATERIALIZE
         size += align - 1
-        malloced = asmmemmgr.malloc(size, size)
+        malloced = cpu.asmmemmgr.malloc(size, size)
         allblocks.append(malloced)
         rawstart = malloced[0]
         rawstart = (rawstart + align - 1) & (-align)
@@ -364,8 +319,9 @@
             assert gcrootmap is not None
             for pos, mark in self.gcroot_markers:
                 gcrootmap.register_asm_addr(rawstart + pos, mark)
-        asmmemmgr.register_frame_depth_map(rawstart, self.frame_positions,
-                                           self.frame_assignments)
+        cpu.codemap.register_frame_depth_map(rawstart, rawstart + size,
+                                             self.frame_positions,
+                                             self.frame_assignments)
         self.frame_positions = None
         self.frame_assignments = None
         return rawstart
diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -130,7 +130,7 @@
         self.gcmap_for_finish[0] = r_uint(1)
 
     def setup(self, looptoken):
-        self.codemap = CodemapBuilder()
+        self.codemap_builder = CodemapBuilder()
         self._finish_gcmap = lltype.nullptr(jitframe.GCMAP)
 
     def set_debug(self, v):
@@ -200,7 +200,9 @@
         return fail_descr, target
 
     def debug_merge_point(self, op):
-        self.codemap.debug_merge_point(op, self.mc.get_relative_pos())
+        self.codemap_builder.debug_merge_point(op.getarg(1).getint(),
+                                               op.getarg(3).getint(),
+                                               self.mc.get_relative_pos())
 
     def call_assembler(self, op, guard_op, argloc, vloc, result_loc, tmploc):
         self._store_force_index(guard_op)
diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py
--- a/rpython/jit/backend/llsupport/codemap.py
+++ b/rpython/jit/backend/llsupport/codemap.py
@@ -9,79 +9,129 @@
 
 """
 
+import os
 from rpython.rlib import rgc
+from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.rlib.entrypoint import jit_entrypoint
-from rpython.jit.backend.llsupport import asmmemmgr
-from rpython.rlib.rbisect import bisect, bisect_tuple
+from rpython.rlib.rbisect import bisect_right, bisect_right_addr
+from rpython.rlib.rbisect import bisect_left, bisect_left_addr
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+from rpython.translator import cdir
 
- at jit_entrypoint([lltype.Signed], lltype.Signed,
-                c_name='pypy_jit_stack_depth_at_loc')
- at rgc.no_collect
-def stack_depth_at_loc(loc):
-    _memmngr = asmmemmgr._memmngr
 
-    pos = bisect(_memmngr.jit_addr_map, loc)
-    if pos == 0 or pos == len(_memmngr.jit_addr_map):
-        return -1
-    return _memmngr.jit_frame_depth_map[pos-1]
+INT_LIST_PTR = rffi.CArrayPtr(lltype.Signed)
 
- at jit_entrypoint([], lltype.Signed, c_name='pypy_jit_start_addr')
-def jit_start_addr():
-    _memmngr = asmmemmgr._memmngr
 
-    return _memmngr.jit_addr_map[0]
+srcdir = os.path.join(os.path.dirname(__file__), 'src')
 
- at jit_entrypoint([], lltype.Signed, c_name='pypy_jit_end_addr')
-def jit_end_addr():
-    _memmngr = asmmemmgr._memmngr
+eci = ExternalCompilationInfo(post_include_bits=["""
+#include 
+RPY_EXTERN long pypy_jit_codemap_add(uintptr_t addr,
+                                     unsigned int machine_code_size,
+                                     long *bytecode_info,
+                                     unsigned int bytecode_info_size);
+RPY_EXTERN long *pypy_jit_codemap_del(uintptr_t addr);
+RPY_EXTERN uintptr_t pypy_jit_codemap_firstkey(void);
+RPY_EXTERN void *pypy_find_codemap_at_addr(long addr, long* start_addr);
+RPY_EXTERN long pypy_yield_codemap_at_addr(void *codemap_raw, long addr,
+                                           long *current_pos_addr);
 
-    return _memmngr.jit_addr_map[-1]
+RPY_EXTERN long pypy_jit_depthmap_add(uintptr_t addr, unsigned int size,
+                                      unsigned int stackdepth);
+RPY_EXTERN void pypy_jit_depthmap_clear(uintptr_t addr, unsigned int size);
 
- at jit_entrypoint([lltype.Signed], lltype.Signed,
-                c_name='pypy_find_codemap_at_addr')
-def find_codemap_at_addr(addr):
-    _memmngr = asmmemmgr._memmngr
+"""], separate_module_sources=[
+    open(os.path.join(srcdir, 'skiplist.c'), 'r').read() +
+    open(os.path.join(srcdir, 'codemap.c'), 'r').read()
+], include_dirs=[cdir])
 
-    res = bisect_tuple(_memmngr.jit_codemap, addr) - 1
-    if res == len(_memmngr.jit_codemap):
-        return -1
-    return res
+def llexternal(name, args, res):
+    return rffi.llexternal(name, args, res, compilation_info=eci,
+                           releasegil=False)
 
- at jit_entrypoint([lltype.Signed, lltype.Signed,
-                 rffi.CArrayPtr(lltype.Signed)], lltype.Signed,
-                 c_name='pypy_yield_codemap_at_addr')
-def yield_bytecode_at_addr(codemap_no, addr, current_pos_addr):
-    """ will return consecutive unique_ids from codemap, starting from position
-    `pos` until addr
+pypy_jit_codemap_add = llexternal('pypy_jit_codemap_add',
+                                  [lltype.Signed, lltype.Signed,
+                                   INT_LIST_PTR, lltype.Signed],
+                                  lltype.Signed)
+pypy_jit_codemap_del = llexternal('pypy_jit_codemap_del',
+                                  [lltype.Signed], INT_LIST_PTR)
+pypy_jit_codemap_firstkey = llexternal('pypy_jit_codemap_firstkey',
+                                       [], lltype.Signed)
+
+pypy_jit_depthmap_add = llexternal('pypy_jit_depthmap_add',
+                                   [lltype.Signed, lltype.Signed,
+                                    lltype.Signed], lltype.Signed)
+pypy_jit_depthmap_clear = llexternal('pypy_jit_depthmap_clear',
+                                     [lltype.Signed, lltype.Signed],
+                                     lltype.Void)
+
+stack_depth_at_loc = llexternal('pypy_jit_stack_depth_at_loc',
+                                [lltype.Signed], lltype.Signed)
+find_codemap_at_addr = llexternal('pypy_find_codemap_at_addr',
+                                  [lltype.Signed, rffi.CArrayPtr(lltype.Signed)], lltype.Signed)
+yield_bytecode_at_addr = llexternal('pypy_yield_codemap_at_addr',
+                                    [lltype.Signed, lltype.Signed,
+                                     rffi.CArrayPtr(lltype.Signed)],
+                                     lltype.Signed)
+
+
+class CodemapStorage(object):
+    """ An immortal wrapper around underlaying jit codemap data
     """
-    _memmngr = asmmemmgr._memmngr
+    def setup(self):
+        if not we_are_translated():
+             # in case someone failed to call free(), in tests only anyway
+             self.free()
 
-    codemap = _memmngr.jit_codemap[codemap_no]
-    current_pos = current_pos_addr[0]
-    start_addr = codemap[0]
-    rel_addr = addr - start_addr
-    while True:
-        if current_pos >= len(codemap[2]):
-            return 0
-        next_start = codemap[2][current_pos + 1]
-        if next_start > rel_addr:
-            return 0
-        next_stop = codemap[2][current_pos + 2]
-        if next_stop > rel_addr:
-            current_pos_addr[0] = current_pos + 4
-            return codemap[2][current_pos]
-        # we need to skip potentially more than one
-        current_pos = codemap[2][current_pos + 3]
+    def free(self):
+        while True:
+            key = pypy_jit_codemap_firstkey()
+            if not key:
+                break
+            items = pypy_jit_codemap_del(key)
+            lltype.free(items, flavor='raw', track_allocation=False)
+
+    def free_asm_block(self, start, stop):
+        items = pypy_jit_codemap_del(start)
+        if items:
+            lltype.free(items, flavor='raw', track_allocation=False)
+        pypy_jit_depthmap_clear(start, stop - start)
+
+    def register_frame_depth_map(self, rawstart, rawstop, frame_positions,
+                                 frame_assignments):
+        if not frame_positions:
+            return
+        assert len(frame_positions) == len(frame_assignments)
+        for i in range(len(frame_positions)-1, -1, -1):
+            pos = rawstart + frame_positions[i]
+            length = rawstop - pos
+            if length > 0:
+                #print "ADD:", pos, length, frame_assignments[i]
+                pypy_jit_depthmap_add(pos, length, frame_assignments[i])
+            rawstop = pos
+
+    def register_codemap(self, (start, size, l)):
+        items = lltype.malloc(INT_LIST_PTR.TO, len(l), flavor='raw',
+                              track_allocation=False)
+        for i in range(len(l)):
+            items[i] = l[i]
+        if pypy_jit_codemap_add(start, size, items, len(l)) < 0:
+            lltype.free(items, flavor='raw', track_allocation=False)
+
+    def finish_once(self):
+        self.free()
 
 def unpack_traceback(addr):
-    codemap_pos = find_codemap_at_addr(addr)
-    assert codemap_pos >= 0
+    codemap_raw = find_codemap_at_addr(addr,
+                                lltype.nullptr(rffi.CArray(lltype.Signed)))
+    if not codemap_raw:
+        return [] # no codemap for that position
     storage = lltype.malloc(rffi.CArray(lltype.Signed), 1, flavor='raw')
     storage[0] = 0
     res = []
     while True:
-        item = yield_bytecode_at_addr(codemap_pos, addr, storage)
+        item = yield_bytecode_at_addr(codemap_raw, addr, storage)
         if item == 0:
             break
         res.append(item)
@@ -95,14 +145,18 @@
         self.patch_position = []
         self.last_call_depth = -1
 
-    def debug_merge_point(self, op, pos):
-        call_depth = op.getarg(1).getint()
+    def debug_merge_point(self, call_depth, unique_id, pos):
         if call_depth != self.last_call_depth:
-            unique_id = op.getarg(3).getint()
             if unique_id == 0: # uninteresting case
                 return
             assert unique_id & 1 == 0
             if call_depth > self.last_call_depth:
+                assert call_depth == self.last_call_depth + 1
+                # ^^^ It should never be the case that we see
+                # debug_merge_points that suddenly go more than *one*
+                # call deeper than the previous one (unless we're at
+                # the start of a bridge, handled by
+                # inherit_code_from_position()).
                 self.l.append(unique_id)
                 self.l.append(pos) # <- this is a relative pos
                 self.patch_position.append(len(self.l))
@@ -139,4 +193,3 @@
             item = self.l[i * 4 + 3] # end in l
             assert item > 0
         return (addr, size, self.l) # XXX compact self.l
-
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -16,7 +16,7 @@
     FieldDescr, ArrayDescr, CallDescr, InteriorFieldDescr,
     FLAG_POINTER, FLAG_FLOAT)
 from rpython.jit.backend.llsupport.memcpy import memset_fn
-from rpython.jit.backend.llsupport import asmmemmgr
+from rpython.jit.backend.llsupport import asmmemmgr, codemap
 from rpython.rlib.unroll import unrolling_iterable
 
 
@@ -49,7 +49,7 @@
         else:
             self._setup_exception_handling_untranslated()
         self.asmmemmgr = asmmemmgr.AsmMemoryManager()
-        asmmemmgr._memmngr = self.asmmemmgr
+        self.codemap = codemap.CodemapStorage()
         self._setup_frame_realloc(translate_support_code)
         ad = self.gc_ll_descr.getframedescrs(self).arraydescr
         self.signedarraydescr = ad
@@ -79,6 +79,9 @@
     def setup(self):
         pass
 
+    def finish_once(self):
+        self.codemap.finish_once()
+
     def _setup_frame_realloc(self, translate_support_code):
         FUNC_TP = lltype.Ptr(lltype.FuncType([llmemory.GCREF, lltype.Signed],
                                              llmemory.GCREF))
@@ -213,6 +216,7 @@
             for rawstart, rawstop in blocks:
                 self.gc_ll_descr.freeing_block(rawstart, rawstop)
                 self.asmmemmgr.free(rawstart, rawstop)
+                self.codemap.free_asm_block(rawstart, rawstop)
 
     def force(self, addr_of_force_token):
         frame = rffi.cast(jitframe.JITFRAMEPTR, addr_of_force_token)
diff --git a/rpython/jit/backend/llsupport/src/codemap.c b/rpython/jit/backend/llsupport/src/codemap.c
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/llsupport/src/codemap.c
@@ -0,0 +1,204 @@
+#include "src/precommondefs.h"
+
+#ifndef HAS_SKIPLIST
+# error "skiplist.c needs to be included before"
+#endif
+
+volatile int pypy_codemap_currently_invalid = 0;
+
+void pypy_codemap_invalid_set(int value)
+{
+    if (value)
+        __sync_lock_test_and_set(&pypy_codemap_currently_invalid, 1);
+    else
+        __sync_lock_release(&pypy_codemap_currently_invalid);
+}
+
+
+/************************************************************/
+/***  codemap storage                                     ***/
+/************************************************************/
+
+typedef struct {
+    unsigned int machine_code_size;
+    unsigned int bytecode_info_size;
+    long *bytecode_info;
+} codemap_data_t;
+
+static skipnode_t jit_codemap_head;
+
+/*** interface used from codemap.py ***/
+
+RPY_EXTERN
+long pypy_jit_codemap_add(uintptr_t addr, unsigned int machine_code_size,
+                          long *bytecode_info, unsigned int bytecode_info_size)
+{
+    skipnode_t *new = skiplist_malloc(sizeof(codemap_data_t));
+    codemap_data_t *data;
+    if (new == NULL)
+        return -1;   /* too bad */
+
+    new->key = addr;
+    data = (codemap_data_t *)new->data;
+    data->machine_code_size = machine_code_size;
+    data->bytecode_info = bytecode_info;
+    data->bytecode_info_size = bytecode_info_size;
+
+    pypy_codemap_invalid_set(1);
+    skiplist_insert(&jit_codemap_head, new);
+    pypy_codemap_invalid_set(0);
+    return 0;
+}
+
+RPY_EXTERN
+long *pypy_jit_codemap_del(uintptr_t addr)
+{
+    long *result;
+    skipnode_t *node;
+
+    pypy_codemap_invalid_set(1);
+    node = skiplist_remove(&jit_codemap_head, addr);
+    pypy_codemap_invalid_set(0);
+
+    if (node == NULL)
+        return NULL;
+    result = ((codemap_data_t *)node->data)->bytecode_info;
+    free(node);
+    return result;
+}
+
+RPY_EXTERN
+uintptr_t pypy_jit_codemap_firstkey(void)
+{
+    return skiplist_firstkey(&jit_codemap_head);
+}
+
+/*** interface used from pypy/module/_vmprof ***/
+
+RPY_EXTERN
+void *pypy_find_codemap_at_addr(long addr, long* start_addr)
+{
+    skipnode_t *codemap = skiplist_search(&jit_codemap_head, addr);
+    codemap_data_t *data;
+    uintptr_t rel_addr;
+
+    if (codemap == &jit_codemap_head) {
+        if (start_addr)
+            *start_addr = 0;
+        return NULL;
+    }
+
+    rel_addr = (uintptr_t)addr - codemap->key;
+    data = (codemap_data_t *)codemap->data;
+    if (rel_addr >= data->machine_code_size) {
+        if (start_addr)
+            *start_addr = 0;
+        return NULL;
+    }
+
+    if (start_addr)
+        *start_addr = (long)codemap->key;
+    return (void *)codemap;
+}
+
+RPY_EXTERN
+long pypy_yield_codemap_at_addr(void *codemap_raw, long addr,
+                                long *current_pos_addr)
+{
+    // will return consecutive unique_ids from codemap, starting from position
+    // `pos` until addr
+    skipnode_t *codemap = (skipnode_t *)codemap_raw;
+    long current_pos = *current_pos_addr;
+    long rel_addr = addr - codemap->key;
+    long next_start, next_stop;
+    codemap_data_t *data = (codemap_data_t *)codemap->data;
+
+    while (1) {
+        if (current_pos >= data->bytecode_info_size)
+            return 0;
+        next_start = data->bytecode_info[current_pos + 1];
+        if (next_start > rel_addr)
+            return 0;
+        next_stop = data->bytecode_info[current_pos + 2];
+        if (next_stop > rel_addr) {
+            *current_pos_addr = current_pos + 4;
+            return data->bytecode_info[current_pos];
+        }
+        // we need to skip potentially more than one
+        current_pos = data->bytecode_info[current_pos + 3];
+    }
+}
+
+/************************************************************/
+/***  depthmap storage                                    ***/
+/************************************************************/
+
+typedef struct {
+    unsigned int block_size;
+    unsigned int stack_depth;
+} depthmap_data_t;
+
+static skipnode_t jit_depthmap_head;
+
+/*** interface used from codemap.py ***/
+
+RPY_EXTERN
+long pypy_jit_depthmap_add(uintptr_t addr, unsigned int size,
+                           unsigned int stackdepth)
+{
+    skipnode_t *new = skiplist_malloc(sizeof(depthmap_data_t));
+    depthmap_data_t *data;
+    if (new == NULL)
+        return -1;   /* too bad */
+
+    new->key = addr;
+    data = (depthmap_data_t *)new->data;
+    data->block_size = size;
+    data->stack_depth = stackdepth;
+
+    pypy_codemap_invalid_set(1);
+    skiplist_insert(&jit_depthmap_head, new);
+    pypy_codemap_invalid_set(0);
+    return 0;
+}
+
+RPY_EXTERN
+void pypy_jit_depthmap_clear(uintptr_t addr, unsigned int size)
+{
+    uintptr_t search_key = addr + size - 1;
+    if (size == 0)
+        return;
+
+    pypy_codemap_invalid_set(1);
+    while (1) {
+        /* search for all nodes belonging to the range, and remove them */
+        skipnode_t *node = skiplist_search(&jit_depthmap_head, search_key);
+        if (node->key < addr)
+            break;   /* exhausted */
+        skiplist_remove(&jit_depthmap_head, node->key);
+        free(node);
+    }
+    pypy_codemap_invalid_set(0);
+}
+
+/*** interface used from pypy/module/_vmprof ***/
+
+RPY_EXTERN
+long pypy_jit_stack_depth_at_loc(long loc)
+{
+    skipnode_t *depthmap = skiplist_search(&jit_depthmap_head, (uintptr_t)loc);
+    depthmap_data_t *data;
+    uintptr_t rel_addr;
+
+    if (depthmap == &jit_depthmap_head)
+        return -1;
+
+    rel_addr = (uintptr_t)loc - depthmap->key;
+    data = (codemap_data_t *)depthmap->data;
+    if (rel_addr >= data->block_size)
+        return -1;
+
+    return data->stack_depth;
+}
+
+/************************************************************/
diff --git a/rpython/jit/backend/llsupport/src/skiplist.c b/rpython/jit/backend/llsupport/src/skiplist.c
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/llsupport/src/skiplist.c
@@ -0,0 +1,103 @@
+#include 
+#include 
+
+#define HAS_SKIPLIST
+#define SKIPLIST_HEIGHT   8
+
+typedef struct skipnode_s {
+    uintptr_t key;
+    char *data;
+    struct skipnode_s *next[SKIPLIST_HEIGHT];   /* may be smaller */
+} skipnode_t;
+
+static skipnode_t *skiplist_malloc(uintptr_t datasize)
+{
+    char *result;
+    uintptr_t basesize;
+    uintptr_t length = 1;
+    while (length < SKIPLIST_HEIGHT && (rand() & 3) == 0)
+        length++;
+    basesize = sizeof(skipnode_t) -
+               (SKIPLIST_HEIGHT - length) * sizeof(skipnode_t *);
+    result = malloc(basesize + datasize);
+    if (result != NULL) {
+        ((skipnode_t *)result)->data = result + basesize;
+    }
+    return (skipnode_t *)result;
+}
+
+static skipnode_t *skiplist_search(skipnode_t *head, uintptr_t searchkey)
+{
+    /* Returns the skipnode with key closest (but <=) searchkey.
+       Note that if there is no item with key <= searchkey in the list,
+       this will return the head node. */
+    uintptr_t level = SKIPLIST_HEIGHT - 1;
+    while (1) {
+        skipnode_t *next = head->next[level];
+        if (next != NULL && next->key <= searchkey) {
+            head = next;
+        }
+        else {
+            if (level == 0)
+                break;
+            level -= 1;
+        }
+    }
+    return head;
+}
+
+static void skiplist_insert(skipnode_t *head, skipnode_t *new)
+{
+    uintptr_t size0 = sizeof(skipnode_t) -
+                      SKIPLIST_HEIGHT * sizeof(skipnode_t *);
+    uintptr_t height_of_new = (new->data - ((char *)new + size0)) /
+                              sizeof(skipnode_t *);
+
+    uintptr_t level = SKIPLIST_HEIGHT - 1;
+    uintptr_t searchkey = new->key;
+    while (1) {
+        skipnode_t *next = head->next[level];
+        if (next != NULL && next->key <= searchkey) {
+            head = next;
+        }
+        else {
+            if (level < height_of_new) {
+                new->next[level] = next;
+                head->next[level] = new;
+                if (level == 0)
+                    break;
+            }
+            level -= 1;
+        }
+    }
+}
+
+static skipnode_t *skiplist_remove(skipnode_t *head, uintptr_t exact_key)
+{
+    uintptr_t level = SKIPLIST_HEIGHT - 1;
+    while (1) {
+        skipnode_t *next = head->next[level];
+        if (next != NULL && next->key <= exact_key) {
+            if (next->key == exact_key) {
+                head->next[level] = next->next[level];
+                if (level == 0)
+                    return next;    /* successfully removed */
+                level -= 1;
+            }
+            else
+                head = next;
+        }
+        else {
+            if (level == 0)
+                return NULL;    /* 'exact_key' not found! */
+            level -= 1;
+        }
+    }
+}
+
+static uintptr_t skiplist_firstkey(skipnode_t *head)
+{
+    if (head->next[0] == NULL)
+        return 0;
+    return head->next[0]->key;
+}
diff --git a/rpython/jit/backend/llsupport/test/test_asmmemmgr.py b/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
--- a/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
@@ -2,7 +2,7 @@
 from rpython.jit.backend.llsupport.asmmemmgr import AsmMemoryManager
 from rpython.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
 from rpython.jit.backend.llsupport.asmmemmgr import BlockBuilderMixin
-from rpython.jit.backend.llsupport import asmmemmgr
+from rpython.jit.backend.llsupport.codemap import CodemapStorage
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib import debug
 
@@ -96,20 +96,21 @@
 class TestAsmMemoryManager:
 
     def setup_method(self, _):
-        self.memmgr = AsmMemoryManager(min_fragment=8,
+        self.asmmemmgr = AsmMemoryManager(min_fragment=8,
                                        num_indices=10,
                                        large_alloc_size=8192)
+        self.codemap = CodemapStorage()
 
     def teardown_method(self, _):
-        self.memmgr._delete()
+        self.asmmemmgr._delete()
 
     def test_malloc_simple(self):
         for i in range(100):
-            while self.memmgr.total_memory_allocated < 16384:
+            while self.asmmemmgr.total_memory_allocated < 16384:
                 reqsize = random.randrange(1, 200)
-                (start, stop) = self.memmgr.malloc(reqsize, reqsize)
+                (start, stop) = self.asmmemmgr.malloc(reqsize, reqsize)
                 assert reqsize <= stop - start < reqsize + 8
-                assert self.memmgr.total_memory_allocated in [8192, 16384]
+                assert self.asmmemmgr.total_memory_allocated in [8192, 16384]
             self.teardown_method(None)
             self.setup_method(None)
 
@@ -123,7 +124,7 @@
             if got and (random.random() < 0.4 or len(got) == 1000):
                 # free
                 start, stop = got.pop(random.randrange(0, len(got)))
-                self.memmgr.free(start, stop)
+                self.asmmemmgr.free(start, stop)
                 real_use -= (stop - start)
                 assert real_use >= 0
             #
@@ -134,18 +135,18 @@
                     reqmaxsize = reqsize
                 else:
                     reqmaxsize = reqsize + random.randrange(0, 200)
-                (start, stop) = self.memmgr.malloc(reqsize, reqmaxsize)
+                (start, stop) = self.asmmemmgr.malloc(reqsize, reqmaxsize)
                 assert reqsize <= stop - start < reqmaxsize + 8
                 for otherstart, otherstop in got:           # no overlap
                     assert otherstop <= start or stop <= otherstart
                 got.append((start, stop))
                 real_use += (stop - start)
-                if self.memmgr.total_memory_allocated == prev_total:
+                if self.asmmemmgr.total_memory_allocated == prev_total:
                     iterations_without_allocating_more += 1
                     if iterations_without_allocating_more == 40000:
                         break    # ok
                 else:
-                    new_total = self.memmgr.total_memory_allocated
+                    new_total = self.asmmemmgr.total_memory_allocated
                     iterations_without_allocating_more = 0
                     print real_use, new_total
                     # We seem to never see a printed value greater
@@ -172,7 +173,7 @@
         #
         gcrootmap = FakeGcRootMap()
         allblocks = []
-        rawstart = mc.materialize(self.memmgr, allblocks, gcrootmap)
+        rawstart = mc.materialize(self, allblocks, gcrootmap)
         p = rffi.cast(rffi.CArrayPtr(lltype.Char), rawstart)
         assert p[0] == 'X'
         assert p[1] == 'x'
@@ -268,16 +269,3 @@
     md.done()
     assert allblocks == [(1597, 1697), (1797, 1835)]
     assert ops == [('free', 1835, 1897)]
-
-def test_find_jit_frame_depth():
-    mgr = AsmMemoryManager()
-    mgr.register_frame_depth_map(11, [0, 5, 10], [1, 2, 3])
-    mgr.register_frame_depth_map(30, [0, 5, 10], [4, 5, 6])
-    mgr.register_frame_depth_map(0, [0, 5, 10], [7, 8, 9])
-    asmmemmgr._memmngr = mgr
-    assert asmmemmgr.stack_depth_at_loc(13) == 1
-    assert asmmemmgr.stack_depth_at_loc(-3) == -1
-    assert asmmemmgr.stack_depth_at_loc(41) == -1
-    assert asmmemmgr.stack_depth_at_loc(5) == 8
-    assert asmmemmgr.stack_depth_at_loc(17) == 2
-    assert asmmemmgr.stack_depth_at_loc(38) == 5
diff --git a/rpython/jit/backend/llsupport/test/test_codemap.py b/rpython/jit/backend/llsupport/test/test_codemap.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/llsupport/test/test_codemap.py
@@ -0,0 +1,93 @@
+
+from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.jit.backend.llsupport.codemap import stack_depth_at_loc
+from rpython.jit.backend.llsupport.codemap import CodemapStorage, \
+     CodemapBuilder, unpack_traceback, find_codemap_at_addr
+
+NULL = lltype.nullptr(rffi.CArray(lltype.Signed))
+     
+def test_register_codemap():
+    codemap = CodemapStorage()
+    codemap.setup()
+    codemap.register_codemap((100, 20, [13, 14, 15]))
+    codemap.register_codemap((300, 30, [16, 17, 18]))
+    codemap.register_codemap((200, 100, [19, 20, 21, 22, 23]))
+    #
+    raw100 = find_codemap_at_addr(100, NULL)
+    assert find_codemap_at_addr(119, NULL) == raw100
+    assert not find_codemap_at_addr(120, NULL)
+    #
+    raw200 = find_codemap_at_addr(200, NULL)
+    assert raw200 != raw100
+    assert find_codemap_at_addr(299, NULL) == raw200
+    #
+    raw300 = find_codemap_at_addr(329, NULL)
+    assert raw300 != raw100 and raw300 != raw200
+    assert find_codemap_at_addr(300, NULL) == raw300
+    #
+    codemap.free()
+
+def test_find_jit_frame_depth():
+    codemap = CodemapStorage()
+    codemap.setup()
+    codemap.register_frame_depth_map(11, 26, [0, 5, 10], [1, 2, 3])
+    codemap.register_frame_depth_map(30, 41, [0, 5, 10], [4, 5, 6])
+    codemap.register_frame_depth_map(0, 11, [0, 5, 10], [7, 8, 9])
+    assert stack_depth_at_loc(13) == 1
+    assert stack_depth_at_loc(-3) == -1
+    assert stack_depth_at_loc(40) == 6
+    assert stack_depth_at_loc(41) == -1
+    assert stack_depth_at_loc(5) == 8
+    assert stack_depth_at_loc(17) == 2
+    assert stack_depth_at_loc(38) == 5
+    assert stack_depth_at_loc(25) == 3
+    assert stack_depth_at_loc(26) == -1
+    assert stack_depth_at_loc(11) == 1
+    assert stack_depth_at_loc(10) == 9
+    codemap.free_asm_block(11, 26)
+    assert stack_depth_at_loc(11) == -1
+    assert stack_depth_at_loc(13) == -1
+    assert stack_depth_at_loc(-3) == -1
+    assert stack_depth_at_loc(40) == 6
+    assert stack_depth_at_loc(41) == -1
+    assert stack_depth_at_loc(5) == 8
+    assert stack_depth_at_loc(38) == 5
+    assert stack_depth_at_loc(10) == 9
+    codemap.free()
+
+def test_codemaps():
+    builder = CodemapBuilder()
+    builder.debug_merge_point(0, 102, 0)
+    builder.debug_merge_point(0, 102, 13)
+    builder.debug_merge_point(1, 104, 15)
+    builder.debug_merge_point(1, 104, 16)
+    builder.debug_merge_point(2, 106, 20)
+    builder.debug_merge_point(2, 106, 25)
+    builder.debug_merge_point(1, 104, 30)
+    builder.debug_merge_point(0, 102, 35)
+    codemap = CodemapStorage()
+    codemap.setup()
+    codemap.register_codemap(builder.get_final_bytecode(100, 40))
+    builder = CodemapBuilder()
+    builder.debug_merge_point(0, 202, 0)
+    builder.debug_merge_point(0, 202, 10)
+    builder.debug_merge_point(1, 204, 20)
+    builder.debug_merge_point(1, 204, 30)
+    builder.debug_merge_point(2, 206, 40)
+    builder.debug_merge_point(2, 206, 50)
+    builder.debug_merge_point(1, 204, 60)
+    builder.debug_merge_point(0, 202, 70)
+    codemap.register_codemap(builder.get_final_bytecode(200, 100))
+    assert unpack_traceback(110) == [102]
+    assert unpack_traceback(117) == [102, 104]
+    assert unpack_traceback(121) == [102, 104, 106]
+    assert unpack_traceback(131) == [102, 104]
+    assert unpack_traceback(137) == [102]
+    assert unpack_traceback(205) == [202]
+    assert unpack_traceback(225) == [202, 204]
+    assert unpack_traceback(245) == [202, 204, 206]
+    assert unpack_traceback(265) == [202, 204]
+    assert unpack_traceback(275) == [202]
+    codemap.free_asm_block(200, 300)
+    assert unpack_traceback(225) == []

From noreply at buildbot.pypy.org  Thu Apr 16 10:17:54 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 16 Apr 2015 10:17:54 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof: close branch rebranded to vmprof2
Message-ID: <20150416081754.02A001C071E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r76803:5fdfa88d2d01
Date: 2015-04-16 10:17 +0200
http://bitbucket.org/pypy/pypy/changeset/5fdfa88d2d01/

Log:	close branch rebranded to vmprof2


From noreply at buildbot.pypy.org  Thu Apr 16 10:17:55 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 16 Apr 2015 10:17:55 +0200 (CEST)
Subject: [pypy-commit] pypy default: dummy merge
Message-ID: <20150416081755.1C53B1C071E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r76804:73bdfba07500
Date: 2015-04-16 10:17 +0200
http://bitbucket.org/pypy/pypy/changeset/73bdfba07500/

Log:	dummy merge


From noreply at buildbot.pypy.org  Thu Apr 16 10:32:02 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 16 Apr 2015 10:32:02 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Killing OP_TYPENAME was a bad idea.
	Test and fix.
Message-ID: <20150416083202.5303F1C1331@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1727:228423fdb5a5
Date: 2015-04-16 10:32 +0200
http://bitbucket.org/cffi/cffi/changeset/228423fdb5a5/

Log:	Killing OP_TYPENAME was a bad idea. Test and fix.

diff --git a/new/cffi_opcode.py b/new/cffi_opcode.py
--- a/new/cffi_opcode.py
+++ b/new/cffi_opcode.py
@@ -27,6 +27,7 @@
 OP_CPYTHON_BLTN_V  = 23   # varargs
 OP_CPYTHON_BLTN_N  = 25   # noargs
 OP_CPYTHON_BLTN_O  = 27   # O  (i.e. a single arg)
+OP_TYPENAME        = 29
 
 PRIM_VOID          = 0
 PRIM_BOOL          = 1
diff --git a/new/parse_c_type.c b/new/parse_c_type.c
--- a/new/parse_c_type.c
+++ b/new/parse_c_type.c
@@ -568,8 +568,7 @@
         {
             int n = search_in_typenames(tok->info->ctx, tok->p, tok->size);
             if (n >= 0) {
-                t1 = _CFFI_OP(_CFFI_OP_NOOP,
-                              tok->info->ctx->typenames[n].type_index);
+                t1 = _CFFI_OP(_CFFI_OP_TYPENAME, n);
                 break;
             }
             n = search_standard_typename(tok->p, tok->size);
diff --git a/new/parse_c_type.h b/new/parse_c_type.h
--- a/new/parse_c_type.h
+++ b/new/parse_c_type.h
@@ -21,6 +21,7 @@
 #define _CFFI_OP_CPYTHON_BLTN_V 23   // varargs
 #define _CFFI_OP_CPYTHON_BLTN_N 25   // noargs
 #define _CFFI_OP_CPYTHON_BLTN_O 27   // O  (i.e. a single arg)
+#define _CFFI_OP_TYPENAME       29
 
 #define _CFFI_PRIM_VOID          0
 #define _CFFI_PRIM_BOOL          1
diff --git a/new/realize_c_type.c b/new/realize_c_type.c
--- a/new/realize_c_type.c
+++ b/new/realize_c_type.c
@@ -190,6 +190,16 @@
         x = _realize_c_type_or_func(ctx, opcodes, _CFFI_GETARG(op));
         break;
 
+    case _CFFI_OP_TYPENAME:
+    {
+        /* essential: the TYPENAME opcode resolves the type index looked
+           up in the 'ctx->typenames' array, but it does so in 'ctx->types'
+           instead of in 'opcodes'! */
+        int type_index = ctx->typenames[_CFFI_GETARG(op)].type_index;
+        x = _realize_c_type_or_func(ctx, ctx->types, type_index);
+        break;
+    }
+
     default:
         PyErr_Format(PyExc_NotImplementedError, "op=%d", (int)_CFFI_GETOP(op));
         return NULL;
diff --git a/new/recompiler.py b/new/recompiler.py
--- a/new/recompiler.py
+++ b/new/recompiler.py
@@ -431,11 +431,15 @@
     outputfilename = ffiplatform.compile(tmpdir, ext)
     return outputfilename
 
-def verify(ffi, module_name, preamble, *args, **kwds):
+def verify2(ffi, module_name, preamble, *args, **kwds):
     import imp
     assert module_name not in sys.modules, "module name conflict: %r" % (
         module_name,)
     outputfilename = recompile(ffi, module_name, preamble, *args, **kwds)
     module = imp.load_dynamic(module_name, outputfilename)
-    ffi._verified(module.ffi)
-    return module.lib
+    return module.ffi, module.lib
+
+def verify(ffi, module_name, preamble, *args, **kwds):
+    ffi2, lib = verify2(ffi, module_name, preamble, *args, **kwds)
+    ffi._verified(ffi2)
+    return lib
diff --git a/new/test_parse_c_type.py b/new/test_parse_c_type.py
--- a/new/test_parse_c_type.py
+++ b/new/test_parse_c_type.py
@@ -93,6 +93,7 @@
 Func = make_getter('FUNCTION')
 FuncEnd = make_getter('FUNCTION_END')
 Struct = make_getter('STRUCT_UNION')
+Typename = make_getter('TYPENAME')
 
 
 def test_simple():
@@ -239,8 +240,8 @@
 
 def test_identifier():
     for i in range(len(identifier_names)):
-        assert parse("%s" % (identifier_names[i])) == ['->', NoOp(100 + i)]
-        assert parse("%s*" % (identifier_names[i])) == [NoOp(100 + i),
+        assert parse("%s" % (identifier_names[i])) == ['->', Typename(i)]
+        assert parse("%s*" % (identifier_names[i])) == [Typename(i),
                                                         '->', Pointer(0)]
 
 def test_cffi_opcode_sync():
diff --git a/new/test_recompiler.py b/new/test_recompiler.py
--- a/new/test_recompiler.py
+++ b/new/test_recompiler.py
@@ -1,5 +1,5 @@
 import py
-from recompiler import Recompiler, verify
+from recompiler import Recompiler, verify, verify2
 from cffi1 import FFI
 
 
@@ -79,7 +79,8 @@
 def test_verify_typedef():
     ffi = FFI()
     ffi.cdef("typedef int **foo_t;")
-    lib = verify(ffi, 'test_verify_typedef', 'typedef int **foo_t;')
+    ffi2, lib = verify2(ffi, 'test_verify_typedef', 'typedef int **foo_t;')
+    assert ffi2.sizeof("foo_t") == ffi.sizeof("void *")
 
 def test_global_var_int():
     ffi = FFI()

From noreply at buildbot.pypy.org  Thu Apr 16 12:19:12 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 16 Apr 2015 12:19:12 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: More about constants
Message-ID: <20150416101912.4A0F51C12E8@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1728:e5ba5bbdb223
Date: 2015-04-16 12:19 +0200
http://bitbucket.org/cffi/cffi/changeset/e5ba5bbdb223/

Log:	More about constants

diff --git a/new/cffi_opcode.py b/new/cffi_opcode.py
--- a/new/cffi_opcode.py
+++ b/new/cffi_opcode.py
@@ -23,11 +23,13 @@
 OP_FUNCTION_END    = 15
 OP_NOOP            = 17
 OP_BITFIELD        = 19
-OP_INTEGER_CONST   = 21
+OP_TYPENAME        = 21
 OP_CPYTHON_BLTN_V  = 23   # varargs
 OP_CPYTHON_BLTN_N  = 25   # noargs
 OP_CPYTHON_BLTN_O  = 27   # O  (i.e. a single arg)
-OP_TYPENAME        = 29
+OP_CONSTANT        = 29
+OP_CONSTANT_INT    = 31
+OP_GLOBAL_VAR      = 33
 
 PRIM_VOID          = 0
 PRIM_BOOL          = 1
diff --git a/new/ffi_obj.c b/new/ffi_obj.c
--- a/new/ffi_obj.c
+++ b/new/ffi_obj.c
@@ -66,7 +66,6 @@
     if (!ffi->ctx_is_static) {
         const void *mem[] = {ffi->info.ctx->types,
                              ffi->info.ctx->globals,
-                             ffi->info.ctx->constants,
                              ffi->info.ctx->struct_unions,
                              ffi->info.ctx->fields,
                              ffi->info.ctx->enums,
diff --git a/new/lib_obj.c b/new/lib_obj.c
--- a/new/lib_obj.c
+++ b/new/lib_obj.c
@@ -111,9 +111,46 @@
         x = lib_build_cpython_func(lib, g, s, METH_O);
         break;
 
-    case _CFFI_OP_NOOP:
-        /* this is used for global variables, of the exact type specified
-           here */
+    case _CFFI_OP_CONSTANT_INT:
+    {
+        /* a constant integer whose value, in an "unsigned long long",
+           is obtained by calling the function at g->address */
+        unsigned long long value;
+        int neg = ((int(*)(unsigned long long*))g->address)(&value);
+        if (!neg) {
+            if (value <= (unsigned long long)LONG_MAX)
+                x = PyInt_FromLong((long)value);
+            else
+                x = PyLong_FromUnsignedLongLong(value);
+        }
+        else {
+            if ((long long)value >= (long long)LONG_MIN)
+                x = PyInt_FromLong((long)value);
+            else
+                x = PyLong_FromLongLong((long long)value);
+        }
+        break;
+    }
+
+    case _CFFI_OP_CONSTANT:
+    {
+        /* a constant which is not of integer type */
+        char *data;
+        ct = realize_c_type(lib->l_ctx, lib->l_ctx->types,
+                            _CFFI_GETARG(g->type_op));
+        if (ct == NULL)
+            return NULL;
+
+        assert(ct->ct_size > 0);
+        data = alloca(ct->ct_size);
+        ((void(*)(char*))g->address)(data);
+        x = convert_to_object(data, ct);
+        Py_DECREF(ct);
+        break;
+    }
+
+    case _CFFI_OP_GLOBAL_VAR:
+        /* global variable of the exact type specified here */
         ct = realize_c_type(lib->l_ctx, lib->l_ctx->types,
                             _CFFI_GETARG(g->type_op));
         if (ct == NULL)
diff --git a/new/manual.c b/new/manual.c
--- a/new/manual.c
+++ b/new/manual.c
@@ -1,6 +1,10 @@
 #include "_cffi_include.h"
 
 
+#define AA  (42)
+#define BB  (&bb)
+static int bb = 16261;
+
 int foo42(int a, int *b)
 {
     return a - *b;
@@ -79,7 +83,21 @@
   return _cffi_from_c_int(result, int);
 }
 
+static int _cffi_const_AA(unsigned long long *output)
+{
+    *output = (unsigned long long)((AA) << 0);   // integer
+    return (AA) <= 0;
+}
+
+static void _cffi_const_BB(char *output)
+{
+    *(int **)output = BB;
+}
+
 static const struct _cffi_global_s _cffi_globals[] = {
+    { "AA",    &_cffi_const_AA, _CFFI_OP(_CFFI_OP_CONSTANT_INT, 0) },
+    { "BB",    &_cffi_const_BB, _CFFI_OP(_CFFI_OP_CONSTANT, 2) },
+    { "bb",    &bb, _CFFI_OP(_CFFI_OP_VARIABLE, 2) },
     { "foo42", &_cffi_f_foo42, _CFFI_OP(_CFFI_OP_CPYTHON_BLTN_V, 0) },
     { "foo64", &_cffi_f_foo64, _CFFI_OP(_CFFI_OP_CPYTHON_BLTN_O, 4) },
 };
@@ -92,7 +110,7 @@
     NULL,
     NULL,
     NULL,
-    2,  /* num_globals */
+    5,  /* num_globals */
     0,
     0,
     0,
diff --git a/new/parse_c_type.h b/new/parse_c_type.h
--- a/new/parse_c_type.h
+++ b/new/parse_c_type.h
@@ -17,11 +17,13 @@
 #define _CFFI_OP_FUNCTION_END   15
 #define _CFFI_OP_NOOP           17
 #define _CFFI_OP_BITFIELD       19
-#define _CFFI_OP_INTEGER_CONST  21
+#define _CFFI_OP_TYPENAME       21
 #define _CFFI_OP_CPYTHON_BLTN_V 23   // varargs
 #define _CFFI_OP_CPYTHON_BLTN_N 25   // noargs
 #define _CFFI_OP_CPYTHON_BLTN_O 27   // O  (i.e. a single arg)
-#define _CFFI_OP_TYPENAME       29
+#define _CFFI_OP_CONSTANT       29
+#define _CFFI_OP_CONSTANT_INT   31
+#define _CFFI_OP_GLOBAL_VAR     33
 
 #define _CFFI_PRIM_VOID          0
 #define _CFFI_PRIM_BOOL          1
@@ -64,12 +66,6 @@
     _cffi_opcode_t type_op;
 };
 
-struct _cffi_constant_s {
-    const char *name;
-    unsigned long long value;
-    _cffi_opcode_t type_op;
-};
-
 struct _cffi_struct_union_s {
     const char *name;
     size_t size;
@@ -102,13 +98,11 @@
 struct _cffi_type_context_s {
     _cffi_opcode_t *types;
     const struct _cffi_global_s *globals;
-    const struct _cffi_constant_s *constants;
     const struct _cffi_struct_union_s *struct_unions;
     const struct _cffi_field_s *fields;
     const struct _cffi_enum_s *enums;
     const struct _cffi_typename_s *typenames;
     int num_globals;
-    int num_constants;
     int num_struct_unions;
     int num_enums;
     int num_typenames;
diff --git a/new/recompiler.py b/new/recompiler.py
--- a/new/recompiler.py
+++ b/new/recompiler.py
@@ -133,14 +133,13 @@
                 prnt()
         #
         # XXX
-        nums['constant'] = 0
         nums['struct_union'] = 0
         nums['enum'] = 0
         #
         # the declaration of '_cffi_type_context'
         prnt('static const struct _cffi_type_context_s _cffi_type_context = {')
         prnt('  _cffi_types,')
-        ALL_STEPS = ["global", "constant", "struct_union", "enum", "typename"]
+        ALL_STEPS = ["global", "struct_union", "enum", "typename"]
         for step_name in ALL_STEPS:
             if nums[step_name] > 0:
                 prnt('  _cffi_%ss,' % step_name)
@@ -354,10 +353,69 @@
         else:
             meth_kind = 'V'   # 'METH_VARARGS'
         self._lsts["global"].append(
-            '  { "%s", _cffi_f_%s, _CFFI_OP(_CFFI_OP_CPYTHON_BLTN_%s, %d)},'
+            '  { "%s", _cffi_f_%s, _CFFI_OP(_CFFI_OP_CPYTHON_BLTN_%s, %d) },'
             % (name, name, meth_kind, type_index))
 
     # ----------
+    # constants, declared with "static const ..."
+
+    def _generate_cpy_const(self, is_int, name, tp=None, category='const',
+                            check_value=None):
+        assert check_value is None # XXX
+        prnt = self._prnt
+        funcname = '_cffi_%s_%s' % (category, name)
+        if is_int:
+            prnt('static int %s(unsigned long long *o)' % funcname)
+            prnt('{')
+            prnt('  *o = (unsigned long long)((%s) << 0);'
+                 '  /* check that we get an integer */' % (name,))
+            prnt('  return (%s) <= 0;' % (name,))
+            prnt('}')
+        else:
+            prnt('static void %s(char *o)' % funcname)
+            prnt('{')
+            prnt('  *(%s)o = %s;' % (tp.get_c_name('*'), name))
+            prnt('}')
+        prnt()
+
+    def _generate_cpy_constant_collecttype(self, tp, name):
+        is_int = isinstance(tp, model.PrimitiveType) and tp.is_integer_type()
+        if not is_int:
+            self._do_collect_type(tp)
+
+    def _generate_cpy_constant_decl(self, tp, name):
+        is_int = isinstance(tp, model.PrimitiveType) and tp.is_integer_type()
+        self._generate_cpy_const(is_int, name, tp)
+
+    def _generate_cpy_constant_ctx(self, tp, name):
+        is_int = isinstance(tp, model.PrimitiveType) and tp.is_integer_type()
+        if not is_int:
+            type_index = self._typesdict[tp]
+            type_op = '_CFFI_OP(_CFFI_OP_CONSTANT, %d)' % type_index
+        else:
+            type_op = '_CFFI_OP(_CFFI_OP_CONSTANT_INT, 0)'
+        self._lsts["global"].append(
+            '  { "%s", _cffi_const_%s, %s },' % (name, name, type_op))
+
+    # ----------
+    # macros: for now only for integers
+
+    def _generate_cpy_macro_collecttype(self, tp, name):
+        pass
+
+    def _generate_cpy_macro_decl(self, tp, name):
+        if tp == '...':
+            check_value = None
+        else:
+            check_value = tp     # an integer
+        self._generate_cpy_const(True, name, check_value=check_value)
+
+    def _generate_cpy_macro_ctx(self, tp, name):
+        self._lsts["global"].append(
+            '  { "%s", _cffi_const_%s, _CFFI_OP(_CFFI_OP_CONSTANT_INT, 0) },' %
+            (name, name))
+
+    # ----------
     # global variables
 
     def _generate_cpy_variable_collecttype(self, tp, name):
@@ -369,7 +427,7 @@
     def _generate_cpy_variable_ctx(self, tp, name):
         type_index = self._typesdict[tp]
         self._lsts["global"].append(
-            '  { "%s", &%s, _CFFI_OP(_CFFI_OP_NOOP, %d)},'
+            '  { "%s", &%s, _CFFI_OP(_CFFI_OP_GLOBAL_VAR, %d)},'
             % (name, name, type_index))
 
     # ----------
@@ -395,6 +453,8 @@
     def _emit_bytecode_PointerType(self, tp, index):
         self.cffi_types[index] = CffiOp(OP_POINTER, self._typesdict[tp.totype])
 
+    _emit_bytecode_ConstPointerType = _emit_bytecode_PointerType
+
     def _emit_bytecode_FunctionPtrType(self, tp, index):
         raw = tp.as_raw_function()
         self.cffi_types[index] = CffiOp(OP_POINTER, self._typesdict[raw])
diff --git a/new/test_recompiler.py b/new/test_recompiler.py
--- a/new/test_recompiler.py
+++ b/new/test_recompiler.py
@@ -99,12 +99,42 @@
     py.test.raises(AttributeError, "del lib.c")
     py.test.raises(AttributeError, "del lib.foobarbaz")
 
+def test_macro():
+    ffi = FFI()
+    ffi.cdef("#define FOOBAR ...")
+    lib = verify(ffi, 'test_macro', "#define FOOBAR (-6912)")
+    assert lib.FOOBAR == -6912
+    py.test.raises(AttributeError, "lib.FOOBAR = 2")
+
+def test_constant():
+    ffi = FFI()
+    ffi.cdef("static const int FOOBAR;")
+    lib = verify(ffi, 'test_constant', "#define FOOBAR (-6912)")
+    assert lib.FOOBAR == -6912
+    py.test.raises(AttributeError, "lib.FOOBAR = 2")
+
+def test_constant_nonint():
+    ffi = FFI()
+    ffi.cdef("static const double FOOBAR;")
+    lib = verify(ffi, 'test_constant_nonint', "#define FOOBAR (-6912.5)")
+    assert lib.FOOBAR == -6912.5
+    py.test.raises(AttributeError, "lib.FOOBAR = 2")
+
+def test_constant_ptr():
+    ffi = FFI()
+    ffi.cdef("static double *const FOOBAR;")
+    lib = verify(ffi, 'test_constant_ptr', "#define FOOBAR NULL")
+    py.test.skip("XXX in-progress:")
+    assert lib.FOOBAR == ffi.NULL
+    assert ffi.typeof(lib.FOOBAR) == ffi.typeof("double *")
+
 def test_dir():
     ffi = FFI()
-    ffi.cdef("int ff(int); int aa;")
+    ffi.cdef("int ff(int); int aa; static const int my_constant;")
     lib = verify(ffi, 'test_dir', """
+        #define my_constant  (-45)
         int aa;
         int ff(int x) { return x+aa; }
     """)
     lib.aa = 5
-    assert dir(lib) == ['aa', 'ff']
+    assert dir(lib) == ['aa', 'ff', 'my_constant']

From noreply at buildbot.pypy.org  Thu Apr 16 15:17:35 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 16 Apr 2015 15:17:35 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: structs and unions, step 1
Message-ID: <20150416131735.E74FE1C1027@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1729:dc74273884b6
Date: 2015-04-16 13:01 +0200
http://bitbucket.org/cffi/cffi/changeset/dc74273884b6/

Log:	structs and unions, step 1

diff --git a/new/recompiler.py b/new/recompiler.py
--- a/new/recompiler.py
+++ b/new/recompiler.py
@@ -42,6 +42,15 @@
                     self.cffi_types.append('LEN') # placeholder
         assert None not in self._typesdict.values()
         #
+        # collect all structs and unions
+        self._struct_unions = {}
+        for tp in all_decls:
+            if isinstance(tp, model.StructOrUnion):
+                self._struct_unions[tp] = None
+        for i, tp in enumerate(sorted(self._struct_unions,
+                                      key=lambda tp: tp.name)):
+            self._struct_unions[tp] = i
+        #
         # emit all bytecode sequences now
         for tp in all_decls:
             method = getattr(self, '_emit_bytecode_' + tp.__class__.__name__)
@@ -118,9 +127,10 @@
         #
         # the declaration of '_cffi_globals' and '_cffi_typenames'
         nums = {}
-        self._lsts = {"global": [], "typename": []}
+        self._lsts = {"global": [], "typename": [],
+                      "struct_union": [], "enum": []}
         self._generate("ctx")
-        for step_name in ["global", "typename"]:
+        for step_name in ["global", "typename", "struct_union", "enum"]:
             lst = self._lsts[step_name]
             nums[step_name] = len(lst)
             if nums[step_name] > 0:
@@ -132,10 +142,6 @@
                 prnt('};')
                 prnt()
         #
-        # XXX
-        nums['struct_union'] = 0
-        nums['enum'] = 0
-        #
         # the declaration of '_cffi_type_context'
         prnt('static const struct _cffi_type_context_s _cffi_type_context = {')
         prnt('  _cffi_types,')
@@ -357,6 +363,15 @@
             % (name, name, meth_kind, type_index))
 
     # ----------
+    # named structs or unions
+
+    def _generate_cpy_struct_collecttype(self, tp, name):
+        for tp1 in tp.fldtypes:
+            self._do_collect_type(tp1)
+
+    _generate_cpy_union_collecttype = _generate_cpy_struct_collecttype
+
+    # ----------
     # constants, declared with "static const ..."
 
     def _generate_cpy_const(self, is_int, name, tp=None, category='const',
@@ -468,6 +483,10 @@
             self.cffi_types[index] = CffiOp(OP_ARRAY, item_index)
             self.cffi_types[index + 1] = CffiOp(None, '%d' % (tp.length,))
 
+    def _emit_bytecode_StructType(self, tp, index):
+        struct_index = self._struct_unions[tp]
+        self.cffi_types[index] = CffiOp(OP_STRUCT_UNION, struct_index)
+
 def make_c_source(ffi, module_name, preamble, target_c_file):
     recompiler = Recompiler(ffi, module_name)
     recompiler.collect_type_table()
diff --git a/new/test_recompiler.py b/new/test_recompiler.py
--- a/new/test_recompiler.py
+++ b/new/test_recompiler.py
@@ -60,6 +60,20 @@
     check_type_table("int32_t f(void);",
                      "(FUNCTION 2)(FUNCTION_END 0)(PRIMITIVE 21)")
 
+def test_struct():
+    check_type_table("struct foo_s { int a; long b; };",
+                     "(PRIMITIVE 7)(PRIMITIVE 9)")
+
+def test_union():
+    check_type_table("union foo_u { int a; long b; };",
+                     "(PRIMITIVE 7)(PRIMITIVE 9)")
+
+def test_struct_used():
+    check_type_table("struct foo_s { int a; long b; }; int f(struct foo_s*);",
+                     "(FUNCTION 3)(POINTER 5)(FUNCTION_END 0)"
+                     "(PRIMITIVE 7)(PRIMITIVE 9)"
+                     "(STRUCT_UNION 0)")
+
 
 def test_math_sin():
     import math

From noreply at buildbot.pypy.org  Thu Apr 16 15:17:37 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 16 Apr 2015 15:17:37 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: in-progress
Message-ID: <20150416131737.173591C1027@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1730:d6e79e8e8194
Date: 2015-04-16 14:26 +0200
http://bitbucket.org/cffi/cffi/changeset/d6e79e8e8194/

Log:	in-progress

diff --git a/new/manual.c b/new/manual.c
--- a/new/manual.c
+++ b/new/manual.c
@@ -15,6 +15,10 @@
     return ~a;
 }
 
+struct foo_s {
+    int a;
+};
+
 /************************************************************/
 
 static void *_cffi_types[] = {
@@ -97,22 +101,35 @@
 static const struct _cffi_global_s _cffi_globals[] = {
     { "AA",    &_cffi_const_AA, _CFFI_OP(_CFFI_OP_CONSTANT_INT, 0) },
     { "BB",    &_cffi_const_BB, _CFFI_OP(_CFFI_OP_CONSTANT, 2) },
-    { "bb",    &bb, _CFFI_OP(_CFFI_OP_VARIABLE, 2) },
+    { "bb",    &bb, _CFFI_OP(_CFFI_OP_GLOBAL_VAR, 1) },
     { "foo42", &_cffi_f_foo42, _CFFI_OP(_CFFI_OP_CPYTHON_BLTN_V, 0) },
     { "foo64", &_cffi_f_foo64, _CFFI_OP(_CFFI_OP_CPYTHON_BLTN_O, 4) },
 };
 
+struct _cffi_align_foo_s { char x; struct foo_s y; };
+
+static const struct _cffi_struct_union_s _cffi_struct_unions[] = {
+    { "foo_s",
+      sizeof(struct foo_s),
+      offsetof(struct _cffi_align_foo_s, y),
+      0,
+      1, 0 },
+};
+
+static const struct _cffi_field_s _cffi_fields[] = {
+    { "a", offsetof(struct foo_s, a), sizeof(((struct foo_s *)0)->a),
+      _CFFI_OP(_CFFI_OP_NOOP, 1) },
+};
+
 static const struct _cffi_type_context_s _cffi_type_context = {
     _cffi_types,
     _cffi_globals,
-    NULL,  /* no constants */
-    NULL,
-    NULL,
+    _cffi_struct_unions,
+    _cffi_fields,
     NULL,
     NULL,
     5,  /* num_globals */
-    0,
-    0,
+    1,  /* num_struct_unions */
     0,
     0,
 };
diff --git a/new/recompiler.py b/new/recompiler.py
--- a/new/recompiler.py
+++ b/new/recompiler.py
@@ -369,6 +369,13 @@
         for tp1 in tp.fldtypes:
             self._do_collect_type(tp1)
 
+    def _generate_cpy_struct_decl(self, tp, name):
+        pass
+
+    def _generate_cpy_struct_ctx(self, tp, name):
+        self._lsts["struct_union"].append(
+            '  { "%s",  ')
+
     _generate_cpy_union_collecttype = _generate_cpy_struct_collecttype
 
     # ----------
diff --git a/new/test_recompiler.py b/new/test_recompiler.py
--- a/new/test_recompiler.py
+++ b/new/test_recompiler.py
@@ -152,3 +152,14 @@
     """)
     lib.aa = 5
     assert dir(lib) == ['aa', 'ff', 'my_constant']
+
+def test_verify_struct():
+    ffi = FFI()
+    ffi.cdef("struct foo_s { int b; short a; };")
+    lib = verify(ffi, 'test_verify_struct',
+                 "struct foo_s { short a; int b; };")
+    p = ffi.new("struct foo_s *", {'a': -32768, 'b': -2147483648})
+    assert p.a == -32768
+    assert p.b == -2147483648
+    py.test.raises(OverflowError, "p.a -= 1")
+    py.test.raises(OverflowError, "p.b -= 1")

From noreply at buildbot.pypy.org  Thu Apr 16 15:17:38 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 16 Apr 2015 15:17:38 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Fixes
Message-ID: <20150416131738.1632D1C1027@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1731:90a008af1df4
Date: 2015-04-16 14:41 +0200
http://bitbucket.org/cffi/cffi/changeset/90a008af1df4/

Log:	Fixes

diff --git a/new/ffi_obj.c b/new/ffi_obj.c
--- a/new/ffi_obj.c
+++ b/new/ffi_obj.c
@@ -37,7 +37,7 @@
 
     FFIObject *ffi;
     if (ctx_is_static) {
-        ffi = (FFIObject *)PyObject_New(FFIObject, ffitype);
+        ffi = (FFIObject *)PyObject_GC_New(FFIObject, ffitype);
         /* we don't call PyObject_GC_Track() here: from _cffi_init_module()
            it is not needed, because in this case the ffi object is immortal */
     }
diff --git a/new/lib_obj.c b/new/lib_obj.c
--- a/new/lib_obj.c
+++ b/new/lib_obj.c
@@ -185,6 +185,7 @@
     if (GlobSupport_Check(x)) {
         return read_global_var((GlobSupportObject *)x);
     }
+    Py_INCREF(x);
     return x;
 }
 

From noreply at buildbot.pypy.org  Thu Apr 16 15:17:39 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 16 Apr 2015 15:17:39 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Structs (opaque so far)
Message-ID: <20150416131739.1B9EF1C1027@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1732:ed9bac3dead2
Date: 2015-04-16 15:18 +0200
http://bitbucket.org/cffi/cffi/changeset/ed9bac3dead2/

Log:	Structs (opaque so far)

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -3629,7 +3629,7 @@
     return new_void_type();
 }
 
-static PyObject *_b_struct_or_union_type(const char *name, int flag)
+static PyObject *new_struct_or_union_type(const char *name, int flag)
 {
     int namelen = strlen(name);
     CTypeDescrObject *td = ctypedescr_new(namelen + 1);
@@ -3654,7 +3654,7 @@
     flag = CT_STRUCT;
     if (strcmp(name, "struct _IO_FILE") == 0 || strcmp(name, "FILE") == 0)
         flag |= CT_IS_FILE;
-    return _b_struct_or_union_type(name, flag);
+    return new_struct_or_union_type(name, flag);
 }
 
 static PyObject *b_new_union_type(PyObject *self, PyObject *args)
@@ -3662,7 +3662,7 @@
     char *name;
     if (!PyArg_ParseTuple(args, "s:new_union_type", &name))
         return NULL;
-    return _b_struct_or_union_type(name, CT_UNION);
+    return new_struct_or_union_type(name, CT_UNION);
 }
 
 static CFieldObject *
diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -251,15 +251,11 @@
             self._declare('function ' + decl.name, tp)
         else:
             if isinstance(node, pycparser.c_ast.Struct):
-                # XXX do we need self._declare in any of those?
-                if node.decls is not None:
-                    self._get_struct_union_enum_type('struct', node)
+                self._get_struct_union_enum_type('struct', node)
             elif isinstance(node, pycparser.c_ast.Union):
-                if node.decls is not None:
-                    self._get_struct_union_enum_type('union', node)
+                self._get_struct_union_enum_type('union', node)
             elif isinstance(node, pycparser.c_ast.Enum):
-                if node.values is not None:
-                    self._get_struct_union_enum_type('enum', node)
+                self._get_struct_union_enum_type('enum', node)
             elif not decl.name:
                 raise api.CDefError("construct does not declare any variable",
                                     decl)
diff --git a/new/manual.c b/new/manual.c
--- a/new/manual.c
+++ b/new/manual.c
@@ -29,6 +29,7 @@
     _CFFI_OP(_CFFI_OP_FUNCTION, 1),
     _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_INT),
     _CFFI_OP(_CFFI_OP_FUNCTION_END, 0),
+    _CFFI_OP(_CFFI_OP_STRUCT_UNION, 0),
 };
 
 static PyObject *
@@ -109,10 +110,9 @@
 struct _cffi_align_foo_s { char x; struct foo_s y; };
 
 static const struct _cffi_struct_union_s _cffi_struct_unions[] = {
-    { "foo_s",
+    { "foo_s", 7, 0,
       sizeof(struct foo_s),
       offsetof(struct _cffi_align_foo_s, y),
-      0,
       1, 0 },
 };
 
diff --git a/new/parse_c_type.h b/new/parse_c_type.h
--- a/new/parse_c_type.h
+++ b/new/parse_c_type.h
@@ -68,11 +68,12 @@
 
 struct _cffi_struct_union_s {
     const char *name;
+    int type_index;          // -> _cffi_types, on a OP_STRUCT_UNION
+    int flags;               // CT_UNION?  CT_IS_OPAQUE?
     size_t size;
-    int alignment;
-    int flags;               // CT_UNION?  CT_IS_OPAQUE?
+    size_t alignment;
+    int first_field_index;   // -> _cffi_fields array
     int num_fields;
-    int first_field_index;   // -> _cffi_fields array
 };
 #define CT_UNION            128
 #define CT_IS_OPAQUE        4096
diff --git a/new/realize_c_type.c b/new/realize_c_type.c
--- a/new/realize_c_type.c
+++ b/new/realize_c_type.c
@@ -143,6 +143,25 @@
         Py_DECREF(z);
         break;
 
+    case _CFFI_OP_STRUCT_UNION:
+    {
+        const struct _cffi_struct_union_s *s;
+        _cffi_opcode_t op2;
+
+        s = &ctx->struct_unions[_CFFI_GETARG(op)];
+        op2 = ctx->types[s->type_index];
+        if ((((uintptr_t)op2) & 1) == 0) {
+            x = (PyObject *)op2;
+            Py_INCREF(x);
+        }
+        else {
+            opcodes = ctx->types;
+            index = s->type_index;
+            x = new_struct_or_union_type(s->name, CT_STRUCT);
+        }
+        break;
+    }
+
     case _CFFI_OP_FUNCTION:
     {
         PyObject *fargs;
@@ -207,6 +226,7 @@
 
     if (x != NULL && opcodes == ctx->types) {
         assert((((uintptr_t)x) & 1) == 0);
+        assert((((uintptr_t)opcodes[index]) & 1) == 1);
         Py_INCREF(x);
         opcodes[index] = x;
     }
diff --git a/new/recompiler.py b/new/recompiler.py
--- a/new/recompiler.py
+++ b/new/recompiler.py
@@ -126,11 +126,13 @@
         self._generate("decl")
         #
         # the declaration of '_cffi_globals' and '_cffi_typenames'
+        ALL_STEPS = ["global", "struct_union", "field", "enum", "typename"]
         nums = {}
-        self._lsts = {"global": [], "typename": [],
-                      "struct_union": [], "enum": []}
+        self._lsts = {}
+        for step_name in ALL_STEPS:
+            self._lsts[step_name] = []
         self._generate("ctx")
-        for step_name in ["global", "typename", "struct_union", "enum"]:
+        for step_name in ALL_STEPS:
             lst = self._lsts[step_name]
             nums[step_name] = len(lst)
             if nums[step_name] > 0:
@@ -145,18 +147,14 @@
         # the declaration of '_cffi_type_context'
         prnt('static const struct _cffi_type_context_s _cffi_type_context = {')
         prnt('  _cffi_types,')
-        ALL_STEPS = ["global", "struct_union", "enum", "typename"]
         for step_name in ALL_STEPS:
             if nums[step_name] > 0:
                 prnt('  _cffi_%ss,' % step_name)
-                if step_name == 'struct_union':
-                    prnt('  _cffi_fields,')
             else:
                 prnt('  NULL,  /* no %ss */' % step_name)
-                if step_name == 'struct_union':
-                    prnt('  NULL,  /* no fields */')
         for step_name in ALL_STEPS:
-            prnt('  %d,  /* num_%ss */' % (nums[step_name], step_name))
+            if step_name != "field":
+                prnt('  %d,  /* num_%ss */' % (nums[step_name], step_name))
         prnt('};')
         prnt()
         #
@@ -366,15 +364,29 @@
     # named structs or unions
 
     def _generate_cpy_struct_collecttype(self, tp, name):
-        for tp1 in tp.fldtypes:
-            self._do_collect_type(tp1)
+        self._do_collect_type(tp)
+        if tp.fldtypes is not None:
+            for tp1 in tp.fldtypes:
+                self._do_collect_type(tp1)
 
     def _generate_cpy_struct_decl(self, tp, name):
-        pass
+        if tp.fldtypes is not None:
+            prnt = self._prnt
+            prnt("struct _cffi_align_%s { char x; %s %s y; };" % (
+                name, tp.kind, name))
+            prnt()
 
     def _generate_cpy_struct_ctx(self, tp, name):
+        type_index = self._typesdict[tp]
+        if tp.fldtypes is not None:
+            size_align = ('\n' +
+                '    sizeof(%s %s),\n' % (tp.kind, name) +
+                '    offsetof(struct _cffi_align_%s, y),\n' % (name,) +
+                '    0, 0 },')
+        else:
+            size_align = ' -1, -1, -1, 0 /* opaque */ },'
         self._lsts["struct_union"].append(
-            '  { "%s",  ')
+            '  { "%s", %d, 0,' % (name, type_index) + size_align)
 
     _generate_cpy_union_collecttype = _generate_cpy_struct_collecttype
 
@@ -494,6 +506,8 @@
         struct_index = self._struct_unions[tp]
         self.cffi_types[index] = CffiOp(OP_STRUCT_UNION, struct_index)
 
+    _emit_bytecode_UnionType = _emit_bytecode_StructType
+
 def make_c_source(ffi, module_name, preamble, target_c_file):
     recompiler = Recompiler(ffi, module_name)
     recompiler.collect_type_table()
diff --git a/new/test_recompiler.py b/new/test_recompiler.py
--- a/new/test_recompiler.py
+++ b/new/test_recompiler.py
@@ -60,13 +60,17 @@
     check_type_table("int32_t f(void);",
                      "(FUNCTION 2)(FUNCTION_END 0)(PRIMITIVE 21)")
 
+def test_struct_opaque():
+    check_type_table("struct foo_s;",
+                     "(STRUCT_UNION 0)")
+
 def test_struct():
     check_type_table("struct foo_s { int a; long b; };",
-                     "(PRIMITIVE 7)(PRIMITIVE 9)")
+                     "(PRIMITIVE 7)(PRIMITIVE 9)(STRUCT_UNION 0)")
 
 def test_union():
     check_type_table("union foo_u { int a; long b; };",
-                     "(PRIMITIVE 7)(PRIMITIVE 9)")
+                     "(PRIMITIVE 7)(PRIMITIVE 9)(STRUCT_UNION 0)")
 
 def test_struct_used():
     check_type_table("struct foo_s { int a; long b; }; int f(struct foo_s*);",
@@ -153,7 +157,14 @@
     lib.aa = 5
     assert dir(lib) == ['aa', 'ff', 'my_constant']
 
+def test_verify_opaque_struct():
+    ffi = FFI()
+    ffi.cdef("struct foo_s;")
+    lib = verify(ffi, 'test_verify_opaque_struct', "struct foo_s;")
+    ffi.new("struct foo_s **")
+
 def test_verify_struct():
+    py.test.skip("XXX in-progress:")
     ffi = FFI()
     ffi.cdef("struct foo_s { int b; short a; };")
     lib = verify(ffi, 'test_verify_struct',

From noreply at buildbot.pypy.org  Thu Apr 16 15:41:02 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 16 Apr 2015 15:41:02 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: comments
Message-ID: <20150416134102.AAC211C071E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1733:d06845968a47
Date: 2015-04-16 15:29 +0200
http://bitbucket.org/cffi/cffi/changeset/d06845968a47/

Log:	comments

diff --git a/new/realize_c_type.c b/new/realize_c_type.c
--- a/new/realize_c_type.c
+++ b/new/realize_c_type.c
@@ -155,9 +155,16 @@
             Py_INCREF(x);
         }
         else {
+            x = new_struct_or_union_type(s->name, CT_STRUCT);
+            /* We are going to update the "primary" OP_STRUCT_OR_UNION
+               slot below, which may be the same or a different one as
+               the "current" slot.  If it is a different one, the
+               current slot is not updated.  But in this case, the
+               next time we walk the same current slot, we'll find the
+               'x' object in the primary slot (op2, above) and then we
+               will update the current slot. */
             opcodes = ctx->types;
             index = s->type_index;
-            x = new_struct_or_union_type(s->name, CT_STRUCT);
         }
         break;
     }

From noreply at buildbot.pypy.org  Thu Apr 16 15:41:03 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 16 Apr 2015 15:41:03 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Unions
Message-ID: <20150416134103.B65CE1C071E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1734:2137e0acbd73
Date: 2015-04-16 15:41 +0200
http://bitbucket.org/cffi/cffi/changeset/2137e0acbd73/

Log:	Unions

diff --git a/new/realize_c_type.c b/new/realize_c_type.c
--- a/new/realize_c_type.c
+++ b/new/realize_c_type.c
@@ -155,7 +155,18 @@
             Py_INCREF(x);
         }
         else {
-            x = new_struct_or_union_type(s->name, CT_STRUCT);
+            int flags;
+            char *name = alloca(8 + strlen(s->name));
+            if (s->flags & CT_UNION) {
+                strcpy(name, "union ");
+                flags = CT_UNION;
+            }
+            else {
+                strcpy(name, "struct ");
+                flags = CT_STRUCT;
+            }
+            strcat(name, s->name);
+            x = new_struct_or_union_type(name, flags);
             /* We are going to update the "primary" OP_STRUCT_OR_UNION
                slot below, which may be the same or a different one as
                the "current" slot.  If it is a different one, the
diff --git a/new/recompiler.py b/new/recompiler.py
--- a/new/recompiler.py
+++ b/new/recompiler.py
@@ -378,6 +378,9 @@
 
     def _generate_cpy_struct_ctx(self, tp, name):
         type_index = self._typesdict[tp]
+        flags = '0'
+        if isinstance(tp, model.UnionType):
+            flags = 'CT_UNION'
         if tp.fldtypes is not None:
             size_align = ('\n' +
                 '    sizeof(%s %s),\n' % (tp.kind, name) +
@@ -386,9 +389,11 @@
         else:
             size_align = ' -1, -1, -1, 0 /* opaque */ },'
         self._lsts["struct_union"].append(
-            '  { "%s", %d, 0,' % (name, type_index) + size_align)
+            '  { "%s", %d, %s,' % (name, type_index, flags) + size_align)
 
     _generate_cpy_union_collecttype = _generate_cpy_struct_collecttype
+    _generate_cpy_union_decl = _generate_cpy_struct_decl
+    _generate_cpy_union_ctx = _generate_cpy_struct_ctx
 
     # ----------
     # constants, declared with "static const ..."
diff --git a/new/test_parse_c_type.py b/new/test_parse_c_type.py
--- a/new/test_parse_c_type.py
+++ b/new/test_parse_c_type.py
@@ -238,6 +238,12 @@
         assert parse("%s %s*" % (tag, struct_names[i])) == [Struct(i),
                                                             '->', Pointer(0)]
 
+def test_exchanging_struct_union():
+    parse_error("union %s" % (struct_names[0],),
+                "wrong kind of tag: struct vs union", 6)
+    parse_error("struct %s" % (struct_names[3],),
+                "wrong kind of tag: struct vs union", 7)
+
 def test_identifier():
     for i in range(len(identifier_names)):
         assert parse("%s" % (identifier_names[i])) == ['->', Typename(i)]
diff --git a/new/test_recompiler.py b/new/test_recompiler.py
--- a/new/test_recompiler.py
+++ b/new/test_recompiler.py
@@ -160,8 +160,14 @@
 def test_verify_opaque_struct():
     ffi = FFI()
     ffi.cdef("struct foo_s;")
-    lib = verify(ffi, 'test_verify_opaque_struct', "struct foo_s;")
-    ffi.new("struct foo_s **")
+    ffi, lib = verify2(ffi, 'test_verify_opaque_struct', "struct foo_s;")
+    assert ffi.typeof("struct foo_s").cname == "struct foo_s"
+
+def test_verify_opaque_union():
+    ffi = FFI()
+    ffi.cdef("union foo_s;")
+    ffi, lib = verify2(ffi, 'test_verify_opaque_union', "union foo_s;")
+    assert ffi.typeof("union foo_s").cname == "union foo_s"
 
 def test_verify_struct():
     py.test.skip("XXX in-progress:")

From noreply at buildbot.pypy.org  Thu Apr 16 17:20:10 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 16 Apr 2015 17:20:10 +0200 (CEST)
Subject: [pypy-commit] cffi default: Parse the imposed field offsets as
 Py_ssize_t, in case they don't fit
Message-ID: <20150416152010.C1CE81C071E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r1735:9caeb942edf5
Date: 2015-04-16 16:40 +0200
http://bitbucket.org/cffi/cffi/changeset/9caeb942edf5/

Log:	Parse the imposed field offsets as Py_ssize_t, in case they don't
	fit into an "int".

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -3761,9 +3761,10 @@
     for (i=0; i

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1736:75221f10643d
Date: 2015-04-16 16:41 +0200
http://bitbucket.org/cffi/cffi/changeset/75221f10643d/

Log:	hg merge default

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -3782,9 +3782,10 @@
     for (i=0; i', libraries=lib_m)
+    assert lib.sin(1.23) == math.sin(1.23)
+
 def test_callback_calling_convention():
     py.test.skip("later")
     if sys.platform != 'win32':

From noreply at buildbot.pypy.org  Thu Apr 16 17:20:13 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 16 Apr 2015 17:20:13 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Force the field list of structs lazily.
Message-ID: <20150416152013.234FC1C071E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1737:7709aec1791f
Date: 2015-04-16 17:20 +0200
http://bitbucket.org/cffi/cffi/changeset/7709aec1791f/

Log:	Force the field list of structs lazily.

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -131,6 +131,7 @@
 #define CT_IS_VOID_PTR         524288
 #define CT_WITH_VAR_ARRAY     1048576
 #define CT_IS_UNSIZED_CHAR_A  2097152
+#define CT_LAZY_FIELD_LIST    4194304
 #define CT_PRIMITIVE_ANY  (CT_PRIMITIVE_SIGNED |        \
                            CT_PRIMITIVE_UNSIGNED |      \
                            CT_PRIMITIVE_CHAR |          \
@@ -420,12 +421,21 @@
 static PyObject *
 get_field_name(CTypeDescrObject *ct, CFieldObject *cf);   /* forward */
 
+#define force_lazy_struct(ct)                                           \
+    ((ct)->ct_stuff != NULL ? 1 : do_realize_lazy_struct(ct))
+
+static int do_realize_lazy_struct(CTypeDescrObject *ct);
+/* forward, implemented in realize_c_type.c */
+
 static PyObject *ctypeget_fields(CTypeDescrObject *ct, void *context)
 {
     if (ct->ct_flags & (CT_STRUCT | CT_UNION)) {
         if (!(ct->ct_flags & CT_IS_OPAQUE)) {
             CFieldObject *cf;
-            PyObject *res = PyList_New(0);
+            PyObject *res;
+            if (force_lazy_struct(ct) < 0)
+                return NULL;
+            res = PyList_New(0);
             if (res == NULL)
                 return NULL;
             for (cf = (CFieldObject *)ct->ct_extra;
@@ -1217,6 +1227,9 @@
 {
     const char *expected;
 
+    if (force_lazy_struct(ct) < 0)
+        return -1;
+
     if (ct->ct_flags & CT_UNION) {
         Py_ssize_t n = PyObject_Size(init);
         if (n < 0)
@@ -2220,18 +2233,25 @@
     if (ct->ct_flags & CT_POINTER)
         ct = ct->ct_itemdescr;
 
-    if ((ct->ct_flags & (CT_STRUCT|CT_UNION)) && ct->ct_stuff != NULL) {
-        cf = (CFieldObject *)PyDict_GetItem(ct->ct_stuff, attr);
-        if (cf != NULL) {
-            /* read the field 'cf' */
-            char *data = cd->c_data + cf->cf_offset;
-            if (cf->cf_bitshift == BS_REGULAR)
-                return convert_to_object(data, cf->cf_type);
-            else if (cf->cf_bitshift == BS_EMPTY_ARRAY)
-                return new_simple_cdata(data,
-                    (CTypeDescrObject *)cf->cf_type->ct_stuff);
-            else
-                return convert_to_object_bitfield(data, cf);
+    if (ct->ct_flags & (CT_STRUCT|CT_UNION)) {
+        switch (force_lazy_struct(ct)) {
+        case 1:
+            cf = (CFieldObject *)PyDict_GetItem(ct->ct_stuff, attr);
+            if (cf != NULL) {
+                /* read the field 'cf' */
+                char *data = cd->c_data + cf->cf_offset;
+                if (cf->cf_bitshift == BS_REGULAR)
+                    return convert_to_object(data, cf->cf_type);
+                else if (cf->cf_bitshift == BS_EMPTY_ARRAY)
+                    return new_simple_cdata(data,
+                        (CTypeDescrObject *)cf->cf_type->ct_stuff);
+                else
+                    return convert_to_object_bitfield(data, cf);
+            }
+        case -1:
+            return NULL;
+        default:
+            ;
         }
     }
     return PyObject_GenericGetAttr((PyObject *)cd, attr);
@@ -2246,18 +2266,25 @@
     if (ct->ct_flags & CT_POINTER)
         ct = ct->ct_itemdescr;
 
-    if ((ct->ct_flags & (CT_STRUCT|CT_UNION)) && ct->ct_stuff != NULL) {
-        cf = (CFieldObject *)PyDict_GetItem(ct->ct_stuff, attr);
-        if (cf != NULL) {
-            /* write the field 'cf' */
-            if (value != NULL) {
-                return convert_field_from_object(cd->c_data, cf, value);
+    if (ct->ct_flags & (CT_STRUCT|CT_UNION)) {
+        switch (force_lazy_struct(ct)) {
+        case 1:
+            cf = (CFieldObject *)PyDict_GetItem(ct->ct_stuff, attr);
+            if (cf != NULL) {
+                /* write the field 'cf' */
+                if (value != NULL) {
+                    return convert_field_from_object(cd->c_data, cf, value);
+                }
+                else {
+                    PyErr_SetString(PyExc_AttributeError,
+                                    "cannot delete struct field");
+                    return -1;
+                }
             }
-            else {
-                PyErr_SetString(PyExc_AttributeError,
-                                "cannot delete struct field");
-                return -1;
-            }
+        case -1:
+            return -1;
+        default:
+            ;
         }
     }
     return PyObject_GenericSetAttr((PyObject *)cd, attr, value);
@@ -3639,6 +3666,7 @@
     td->ct_size = -1;
     td->ct_length = -1;
     td->ct_flags = flag | CT_IS_OPAQUE;
+    td->ct_extra = NULL;
     memcpy(td->ct_name, name, namelen + 1);
     td->ct_name_position = namelen;
     return (PyObject *)td;
@@ -4102,6 +4130,8 @@
            here, so better safe (and forbid it) than sorry (and maybe
            crash).
         */
+        if (force_lazy_struct(ct) < 0)
+            return NULL;
         if (ct->ct_flags & CT_CUSTOM_FIELD_POS) {
             PyErr_SetString(PyExc_TypeError,
                 "cannot pass as an argument a struct that was completed "
@@ -4876,10 +4906,12 @@
     if (PyTextAny_Check(fieldname)) {
         if (!following && (ct->ct_flags & CT_POINTER))
             ct = ct->ct_itemdescr;
-        if (!(ct->ct_flags & (CT_STRUCT|CT_UNION)) || ct->ct_stuff == NULL) {
-            PyErr_SetString(PyExc_TypeError,
-                            "with a field name argument, expected an "
-                            "initialized struct or union ctype");
+        if (!(ct->ct_flags & (CT_STRUCT|CT_UNION)) ||
+                force_lazy_struct(ct) <= 0) {
+            if (!PyErr_Occurred())
+                PyErr_SetString(PyExc_TypeError,
+                                "with a field name argument, expected an "
+                                "initialized struct or union ctype");
             return NULL;
         }
         cf = (CFieldObject *)PyDict_GetItem(ct->ct_stuff, fieldname);
diff --git a/new/ffi_obj.c b/new/ffi_obj.c
--- a/new/ffi_obj.c
+++ b/new/ffi_obj.c
@@ -301,7 +301,7 @@
 static CFieldObject *_ffi_field(CTypeDescrObject *ct, const char *fieldname)
 {
     CFieldObject *cf;
-    if (ct->ct_stuff == NULL) {
+    if (force_lazy_struct(ct) == NULL) {
         PyErr_Format(PyExc_TypeError, "'%s' is incomplete", ct->ct_name);
         return NULL;
     }
diff --git a/new/manual.c b/new/manual.c
--- a/new/manual.c
+++ b/new/manual.c
@@ -124,8 +124,8 @@
 static const struct _cffi_type_context_s _cffi_type_context = {
     _cffi_types,
     _cffi_globals,
+    _cffi_fields,
     _cffi_struct_unions,
-    _cffi_fields,
     NULL,
     NULL,
     5,  /* num_globals */
diff --git a/new/parse_c_type.h b/new/parse_c_type.h
--- a/new/parse_c_type.h
+++ b/new/parse_c_type.h
@@ -99,8 +99,8 @@
 struct _cffi_type_context_s {
     _cffi_opcode_t *types;
     const struct _cffi_global_s *globals;
+    const struct _cffi_field_s *fields;
     const struct _cffi_struct_union_s *struct_unions;
-    const struct _cffi_field_s *fields;
     const struct _cffi_enum_s *enums;
     const struct _cffi_typename_s *typenames;
     int num_globals;
@@ -120,3 +120,5 @@
 int parse_c_type(struct _cffi_parse_info_s *info, const char *input);
 int search_in_globals(const struct _cffi_type_context_s *ctx,
                       const char *search, size_t search_len);
+int search_in_struct_unions(const struct _cffi_type_context_s *ctx,
+                            const char *search, size_t search_len);
diff --git a/new/realize_c_type.c b/new/realize_c_type.c
--- a/new/realize_c_type.c
+++ b/new/realize_c_type.c
@@ -167,6 +167,16 @@
             }
             strcat(name, s->name);
             x = new_struct_or_union_type(name, flags);
+
+            if (s->first_field_index >= 0) {
+                CTypeDescrObject *ct = (CTypeDescrObject *)x;
+                ct->ct_size = s->size;
+                ct->ct_length = s->alignment;
+                ct->ct_flags &= ~CT_IS_OPAQUE;
+                ct->ct_flags |= CT_LAZY_FIELD_LIST;
+                ct->ct_extra = (void *)ctx;
+            }
+
             /* We are going to update the "primary" OP_STRUCT_OR_UNION
                slot below, which may be the same or a different one as
                the "current" slot.  If it is a different one, the
@@ -250,3 +260,89 @@
     }
     return x;
 };
+
+static int do_realize_lazy_struct(CTypeDescrObject *ct)
+{
+    assert(ct->ct_flags & (CT_STRUCT | CT_UNION));
+
+    if (ct->ct_flags & CT_LAZY_FIELD_LIST) {
+        assert(!(ct->ct_flags & CT_IS_OPAQUE));
+
+        const struct _cffi_type_context_s *ctx = ct->ct_extra;
+        assert(ctx != NULL);
+
+        char *p = ct->ct_name;
+        if (memcmp(p, "struct ", 7) == 0)
+            p += 7;
+        else if (memcmp(p, "union ", 6) == 0)
+            p += 6;
+
+        int n = search_in_struct_unions(ctx, p, strlen(p));
+        if (n < 0)
+            Py_FatalError("lost a struct/union!");
+
+        const struct _cffi_struct_union_s *s = &ctx->struct_unions[n];
+        const struct _cffi_field_s *fld = &ctx->fields[s->first_field_index];
+
+        /* XXX painfully build all the Python objects that are the args
+           to b_complete_struct_or_union() */
+
+        PyObject *fields = PyList_New(s->num_fields);
+        if (fields == NULL)
+            return -1;
+
+        int i;
+        for (i = 0; i < s->num_fields; i++, fld++) {
+            _cffi_opcode_t op = fld->field_type_op;
+            PyObject *f;
+            CTypeDescrObject *ctf;
+
+            switch (_CFFI_GETOP(op)) {
+
+            case _CFFI_OP_NOOP:
+                ctf = realize_c_type(ctx, ctx->types, _CFFI_GETARG(op));
+                break;
+
+            default:
+                PyErr_Format(PyExc_NotImplementedError, "field op=%d",
+                             (int)_CFFI_GETOP(op));
+                return -1;
+            }
+
+            f = Py_BuildValue("(sOin)", fld->name, ctf,
+                              (int)-1, (Py_ssize_t)fld->field_offset);
+            if (f == NULL) {
+                Py_DECREF(fields);
+                return -1;
+            }
+            PyList_SET_ITEM(fields, i, f);
+        }
+
+        PyObject *args = Py_BuildValue("(OOOnn)", ct, fields,
+                                       Py_None,
+                                       (Py_ssize_t)s->size,
+                                       (Py_ssize_t)s->alignment);
+        Py_DECREF(fields);
+        if (args == NULL)
+            return -1;
+
+        ct->ct_extra = NULL;
+        ct->ct_flags |= CT_IS_OPAQUE;
+        PyObject *res = b_complete_struct_or_union(NULL, args);
+        ct->ct_flags &= ~CT_IS_OPAQUE;
+        Py_DECREF(args);
+        if (res == NULL) {
+            ct->ct_extra = (void *)ctx;
+            return -1;
+        }
+
+        assert(ct->ct_stuff != NULL);
+        ct->ct_flags &= ~CT_LAZY_FIELD_LIST;
+        Py_DECREF(res);
+        return 1;
+    }
+    else {
+        assert(ct->ct_flags & CT_IS_OPAQUE);
+        return 0;
+    }
+}
diff --git a/new/recompiler.py b/new/recompiler.py
--- a/new/recompiler.py
+++ b/new/recompiler.py
@@ -126,7 +126,7 @@
         self._generate("decl")
         #
         # the declaration of '_cffi_globals' and '_cffi_typenames'
-        ALL_STEPS = ["global", "struct_union", "field", "enum", "typename"]
+        ALL_STEPS = ["global", "field", "struct_union", "enum", "typename"]
         nums = {}
         self._lsts = {}
         for step_name in ALL_STEPS:
@@ -139,6 +139,8 @@
                 lst.sort()  # sort by name, which is at the start of each line
                 prnt('static const struct _cffi_%s_s _cffi_%ss[] = {' % (
                     step_name, step_name))
+                if step_name == 'field':
+                    self._fix_final_field_list(lst)
                 for line in lst:
                     prnt(line)
                 prnt('};')
@@ -382,15 +384,35 @@
         if isinstance(tp, model.UnionType):
             flags = 'CT_UNION'
         if tp.fldtypes is not None:
+            c_field = [name]
+            for fldname, fldtype in zip(tp.fldnames, tp.fldtypes):
+                spaces = " " * len(fldname)
+                c_field.append(
+                    '  { "%s", offsetof(%s, %s),\n' % (
+                            fldname, tp.get_c_name(''), fldname) +
+                    '     %s   sizeof(((%s)0)->%s),\n' % (
+                            spaces, tp.get_c_name('*'), fldname) +
+                    '     %s   _CFFI_OP(_CFFI_OP_NOOP, %s) },' % (
+                            spaces, self._typesdict[fldtype]))
+            self._lsts["field"].append('\n'.join(c_field))
             size_align = ('\n' +
                 '    sizeof(%s %s),\n' % (tp.kind, name) +
                 '    offsetof(struct _cffi_align_%s, y),\n' % (name,) +
-                '    0, 0 },')
+                '    _cffi_FIELDS_FOR_%s, %d },' % (name, len(tp.fldtypes),))
         else:
             size_align = ' -1, -1, -1, 0 /* opaque */ },'
         self._lsts["struct_union"].append(
             '  { "%s", %d, %s,' % (name, type_index, flags) + size_align)
 
+    def _fix_final_field_list(self, lst):
+        count = 0
+        for i in range(len(lst)):
+            struct_fields = lst[i]
+            name = struct_fields.split('\n')[0]
+            define_macro = '#define _cffi_FIELDS_FOR_%s  %d' % (name, count)
+            lst[i] = define_macro + struct_fields[len(name):]
+            count += lst[i].count('\n  { "')
+
     _generate_cpy_union_collecttype = _generate_cpy_struct_collecttype
     _generate_cpy_union_decl = _generate_cpy_struct_decl
     _generate_cpy_union_ctx = _generate_cpy_struct_ctx
diff --git a/new/test_recompiler.py b/new/test_recompiler.py
--- a/new/test_recompiler.py
+++ b/new/test_recompiler.py
@@ -170,13 +170,17 @@
     assert ffi.typeof("union foo_s").cname == "union foo_s"
 
 def test_verify_struct():
-    py.test.skip("XXX in-progress:")
     ffi = FFI()
-    ffi.cdef("struct foo_s { int b; short a; };")
+    ffi.cdef("""struct foo_s { int b; short a; };
+                struct bar_s { struct foo_s *f; };""")
     lib = verify(ffi, 'test_verify_struct',
-                 "struct foo_s { short a; int b; };")
+                 """struct foo_s { short a; int b; };
+                    struct bar_s { struct foo_s *f; };""")
     p = ffi.new("struct foo_s *", {'a': -32768, 'b': -2147483648})
     assert p.a == -32768
     assert p.b == -2147483648
     py.test.raises(OverflowError, "p.a -= 1")
     py.test.raises(OverflowError, "p.b -= 1")
+    py.test.skip("XXX in-progress:")
+    q = ffi.new("struct bar_s *", {'f': p})
+    assert q.f == p

From noreply at buildbot.pypy.org  Thu Apr 16 17:52:21 2015
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 16 Apr 2015 17:52:21 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: template for my talk
Message-ID: <20150416155221.7399A1C1230@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: extradoc
Changeset: r5529:6ba8055f6c90
Date: 2015-04-16 16:50 +0200
http://bitbucket.org/pypy/extradoc/changeset/6ba8055f6c90/

Log:	template for my talk

diff --git a/talk/pycon-italy-2015/Makefile b/talk/pycon-italy-2015/Makefile
new file mode 100644
--- /dev/null
+++ b/talk/pycon-italy-2015/Makefile
@@ -0,0 +1,18 @@
+# you can find rst2beamer.py here:
+# http://codespeak.net/svn/user/antocuni/bin/rst2beamer.py
+
+# WARNING: to work, it needs this patch for docutils
+# https://sourceforge.net/tracker/?func=detail&atid=422032&aid=1459707&group_id=38414
+
+talk.pdf: talk.rst author.latex title.latex stylesheet.latex
+	python `which rst2beamer.py` --stylesheet=stylesheet.latex --documentoptions=14pt talk.rst talk.latex || exit
+	#/home/antocuni/.virtualenvs/rst2beamer/bin/python `which rst2beamer.py` --stylesheet=stylesheet.latex --documentoptions=14pt talk.rst talk.latex || exit
+	sed 's/\\date{}/\\input{author.latex}/' -i talk.latex || exit
+	#sed 's/\\maketitle/\\input{title.latex}/' -i talk.latex || exit
+	pdflatex talk.latex  || exit
+
+view: talk.pdf
+	evince talk.pdf &
+
+xpdf: talk.pdf
+	xpdf talk.pdf &
diff --git a/talk/pycon-italy-2015/author.latex b/talk/pycon-italy-2015/author.latex
new file mode 100644
--- /dev/null
+++ b/talk/pycon-italy-2015/author.latex
@@ -0,0 +1,8 @@
+\definecolor{rrblitbackground}{rgb}{0.0, 0.0, 0.0}
+
+\title[PyPy JIT]{PyPy JIT (not) for dummies}
+\author[antocuni]
+{Antonio Cuni}
+
+\institute{PyCon Sei}
+\date{April 17, 2015}
diff --git a/talk/pycon-italy-2015/beamerdefs.txt b/talk/pycon-italy-2015/beamerdefs.txt
new file mode 100644
--- /dev/null
+++ b/talk/pycon-italy-2015/beamerdefs.txt
@@ -0,0 +1,108 @@
+.. colors
+.. ===========================
+
+.. role:: green
+.. role:: red
+
+
+.. general useful commands
+.. ===========================
+
+.. |pause| raw:: latex
+
+   \pause
+
+.. |small| raw:: latex
+
+   {\small
+
+.. |end_small| raw:: latex
+
+   }
+
+.. |scriptsize| raw:: latex
+
+   {\scriptsize
+
+.. |end_scriptsize| raw:: latex
+
+   }
+
+.. |strike<| raw:: latex
+
+   \sout{
+
+.. closed bracket
+.. ===========================
+
+.. |>| raw:: latex
+
+   }
+
+
+.. example block
+.. ===========================
+
+.. |example<| raw:: latex
+
+   \begin{exampleblock}{
+
+
+.. |end_example| raw:: latex
+
+   \end{exampleblock}
+
+
+
+.. alert block
+.. ===========================
+
+.. |alert<| raw:: latex
+
+   \begin{alertblock}{
+
+
+.. |end_alert| raw:: latex
+
+   \end{alertblock}
+
+
+
+.. columns
+.. ===========================
+
+.. |column1| raw:: latex
+
+   \begin{columns}
+      \begin{column}{0.45\textwidth}
+
+.. |column2| raw:: latex
+
+      \end{column}
+      \begin{column}{0.45\textwidth}
+
+
+.. |end_columns| raw:: latex
+
+      \end{column}
+   \end{columns}
+
+
+
+.. |snake| image:: ../../img/py-web-new.png
+           :scale: 15%
+           
+
+
+.. nested blocks
+.. ===========================
+
+.. |nested| raw:: latex
+
+   \begin{columns}
+      \begin{column}{0.85\textwidth}
+
+.. |end_nested| raw:: latex
+
+      \end{column}
+   \end{columns}
diff --git a/talk/pycon-italy-2015/stylesheet.latex b/talk/pycon-italy-2015/stylesheet.latex
new file mode 100644
--- /dev/null
+++ b/talk/pycon-italy-2015/stylesheet.latex
@@ -0,0 +1,10 @@
+\usetheme{Boadilla}
+\setbeamercovered{transparent}
+\setbeamertemplate{navigation symbols}{}
+
+\definecolor{darkgreen}{rgb}{0, 0.5, 0.0}
+\newcommand{\docutilsrolegreen}[1]{\color{darkgreen}#1\normalcolor}
+\newcommand{\docutilsrolered}[1]{\color{red}#1\normalcolor}
+
+\newcommand{\green}[1]{\color{darkgreen}#1\normalcolor}
+\newcommand{\red}[1]{\color{red}#1\normalcolor}
diff --git a/talk/pycon-italy-2015/talk.pdf.info b/talk/pycon-italy-2015/talk.pdf.info
new file mode 100644
--- /dev/null
+++ b/talk/pycon-italy-2015/talk.pdf.info
@@ -0,0 +1,11 @@
+AvailableTransitions=[Crossfade]
+TransitionDuration = 100
+EstimatedDuration = 45*60       # in seconds
+MinutesOnly = True
+
+PageProps = {
+    1: {
+    'reset': FirstTimeOnly,
+    'progress': False,
+    },
+}
diff --git a/talk/pycon-italy-2015/talk.rst b/talk/pycon-italy-2015/talk.rst
new file mode 100644
--- /dev/null
+++ b/talk/pycon-italy-2015/talk.rst
@@ -0,0 +1,41 @@
+.. include:: beamerdefs.txt
+
+================================
+PyPy JIT (not) for dummies
+================================
+
+About me
+---------
+
+- PyPy core dev
+
+- ``pdb++``, ``fancycompleter``, ...
+
+- Consultant, trainer
+
+- http://antocuni.eu
+
+
+PyPy status
+----------------
+
+bla bla
+
+
+Contacts, Q&A
+--------------
+
+- http://pypy.org
+
+- http://morepypy.blogspot.com/
+
+- twitter: @antocuni
+
+- Available for consultancy & training:
+
+  * http://antocuni.eu
+
+  * info at antocuni.eu
+
+- Any question?
+
diff --git a/talk/pycon-italy-2015/title.latex b/talk/pycon-italy-2015/title.latex
new file mode 100644
--- /dev/null
+++ b/talk/pycon-italy-2015/title.latex
@@ -0,0 +1,5 @@
+\begin{titlepage}
+\begin{figure}[h]
+\includegraphics[width=80px]{../img/py-web.png}
+\end{figure}
+\end{titlepage}

From noreply at buildbot.pypy.org  Thu Apr 16 17:52:22 2015
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 16 Apr 2015 17:52:22 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: first part of the slides
Message-ID: <20150416155222.8C73C1C1230@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: extradoc
Changeset: r5530:5d0633b2f4f8
Date: 2015-04-16 17:29 +0200
http://bitbucket.org/pypy/extradoc/changeset/5d0633b2f4f8/

Log:	first part of the slides

diff --git a/talk/pycon-italy-2015/speed.png b/talk/pycon-italy-2015/speed.png
new file mode 100644
index 0000000000000000000000000000000000000000..e2372b9e0d5fc3fe7711b27d641d2d9706185fd3
GIT binary patch

[cut]

diff --git a/talk/pycon-italy-2015/talk.rst b/talk/pycon-italy-2015/talk.rst
--- a/talk/pycon-italy-2015/talk.rst
+++ b/talk/pycon-italy-2015/talk.rst
@@ -16,10 +16,90 @@
 - http://antocuni.eu
 
 
-PyPy status
-----------------
+What is PyPy
+---------------
 
-bla bla
+- Alternative, fast Python implementation
+
+- Performance: JIT compiler, advanced GC
+
+- STM: goodbye GIL
+
+- PyPy 2.5.1 (2.7.8)
+
+- Py3k as usual in progress (3.2.5 out, 3.3 in development)
+
+- http://pypy.org
+
+
+STM
+---
+
+- pypy-stm-2.5.1 is out
+
+  * 64 bit Linux only
+
+- no GIL!
+
+- 25-40% slowdown for single core programs
+
+  * still ``7*0.75 = 5.25x`` faster than CPython :)
+
+- parallelism up to 4 threads
+
+- concurrency slow-but-correct by default
+
+  * compared to fast-but-buggy by using threads
+
+- conflict detection
+
+- TransactionQueue: parallelize your program without using threads!
+
+
+Extension modules
+------------------
+
+- CFFI: stable, mature and widely used
+
+  * psycopg2cffi, lxml-cffi, pysdl2-cffi, etc.
+
+  * should be used even for CPython-only projects!
+
+- numpy:
+
+  * support for linalg
+
+  * support for pure Python, JIT friendly ufuncs
+
+  * object dtype in-progress
+
+- scipy: see next slide :)
+
+
+Pymetabiosis
+--------------
+
+- embed CPython in PyPy
+
+- import and use CPython modules in PyPy
+
+- ALPHA status
+
+- slow when passing arbitrary objects
+
+- but fast for numpy arrays
+
+- matplotlib and scipy works
+
+- https://github.com/rguillebert/pymetabiosis
+
+
+
+Speed: 7x faster than CPython
+-------------------------------
+
+.. image:: speed.png
+   :scale: 47%
 
 
 Contacts, Q&A

From noreply at buildbot.pypy.org  Thu Apr 16 17:52:23 2015
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 16 Apr 2015 17:52:23 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: more slides
Message-ID: <20150416155223.9FB941C1230@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: extradoc
Changeset: r5531:9b1bfdbddfa5
Date: 2015-04-16 17:52 +0200
http://bitbucket.org/pypy/extradoc/changeset/9b1bfdbddfa5/

Log:	more slides

diff --git a/talk/pycon-italy-2015/jit-overview.odg b/talk/pycon-italy-2015/jit-overview.odg
new file mode 100644
index 0000000000000000000000000000000000000000..43aec7c16da3c72bec87a6be52ce41969f4db8c7
GIT binary patch

[cut]

diff --git a/talk/pycon-italy-2015/jit-overview1.pdf b/talk/pycon-italy-2015/jit-overview1.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..6b511590edd06e9deb157eb02cecb7cf64c61926
GIT binary patch

[cut]

diff --git a/talk/pycon-italy-2015/jit-overview2.pdf b/talk/pycon-italy-2015/jit-overview2.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..c07d0e3948403a7c42951a65ba023311d4739ac8
GIT binary patch

[cut]

diff --git a/talk/pycon-italy-2015/jit-overview3.pdf b/talk/pycon-italy-2015/jit-overview3.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..277b2067bc2daf79f7696edaabd22614182e6bfa
GIT binary patch

[cut]

diff --git a/talk/pycon-italy-2015/talk.rst b/talk/pycon-italy-2015/talk.rst
--- a/talk/pycon-italy-2015/talk.rst
+++ b/talk/pycon-italy-2015/talk.rst
@@ -102,6 +102,30 @@
    :scale: 47%
 
 
+The JIT
+--------
+
+.. image:: jit-overview1.pdf
+   :scale: 50%
+
+
+The JIT
+--------
+
+.. image:: jit-overview2.pdf
+   :scale: 50%
+
+
+The JIT
+--------
+
+.. image:: jit-overview3.pdf
+   :scale: 50%
+          
+
+
+
+
 Contacts, Q&A
 --------------
 

From noreply at buildbot.pypy.org  Thu Apr 16 17:52:24 2015
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 16 Apr 2015 17:52:24 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: merge
Message-ID: <20150416155224.B519F1C1230@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: extradoc
Changeset: r5532:2403b934d742
Date: 2015-04-16 17:52 +0200
http://bitbucket.org/pypy/extradoc/changeset/2403b934d742/

Log:	merge

diff --git a/talk/ep2015/performance-abstract.txt b/talk/ep2015/performance-abstract.txt
--- a/talk/ep2015/performance-abstract.txt
+++ b/talk/ep2015/performance-abstract.txt
@@ -1,20 +1,25 @@
-==================================
-PyPy performance (not) for dummies
-==================================
+==============================================
+Python and PyPy performance (not) for dummies
+==============================================
 
+[[The talk will be given by Antonio Cuni and Maciej Fijalkowski,
+both long time PyPy core developers and expert in the area of
+Python performance.]]
 
-Abstract
----------
+In this talk we would like to have a short introduction on how Python
+programs are compiled and executed, with a special attention towards
+just in time compilation done by PyPy. PyPy is the most advanced Python
+interpreter around and while it should generally just speed up your programs
+there is a wide range of performance that you can get out of PyPy, ranging from
+slightly faster than CPython to C speeds, depending on how you write your
+programs.
 
-PyPy is the fastest Python interpreter around, and its JIT can optimize most
-of your Python programs without problems. However, there are techniques to
-improve the performances even further and squeeze the most out of PyPy. In
-this talk we will see:
+We will split the talk in two parts. In the first part we will explain
+how things work and what can and what cannot be optimized as well as describe
+the basic heuristics of JIT compiler and optimizer. In the next part we will
+do a survey of existing tools for looking at performance of Python programs
+with specific focus on PyPy.
 
-- the general principles behind the PyPy JIT
-
-- how to profile programs to find the bottlenecks
-
-- how to examine the code generated by the JIT
-
-- how to write JIT-friendly programs
+As a result of this talk, an audience member should be better equipped with
+tools how to write new software and improve existing software with performance
+in mind.

From noreply at buildbot.pypy.org  Thu Apr 16 19:57:32 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Thu, 16 Apr 2015 19:57:32 +0200 (CEST)
Subject: [pypy-commit] pypy object-dtype2: fix some translation issues
Message-ID: <20150416175732.1EFAF1C1234@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: object-dtype2
Changeset: r76805:79f2ee46aa25
Date: 2015-04-16 18:57 +0100
http://bitbucket.org/pypy/pypy/changeset/79f2ee46aa25/

Log:	fix some translation issues

diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -205,12 +205,9 @@
             w_res = self.getitem_filter(space, w_idx)
         else:
             try:
-                w_res = self.implementation.descr_getitem(space, self, w_idx)
+                return self.implementation.descr_getitem(space, self, w_idx)
             except ArrayArgumentException:
-                w_res = self.getitem_array_int(space, w_idx)
-        if w_res.is_scalar() and w_res.get_dtype(space).is_object():
-            return w_res.get_dtype(space).itemtype.unbox(w_res)
-        return w_res
+                return self.getitem_array_int(space, w_idx)
 
     def getitem(self, space, index_list):
         return self.implementation.getitem_index(space, index_list)
@@ -849,7 +846,7 @@
                         "new type not compatible with array."))
                 # Strides, shape does not change
                 v = impl.astype(space, dtype)
-                return wrap_impl(space, w_type, self, v) 
+                return wrap_impl(space, w_type, self, v)
             strides = impl.get_strides()
             if dims == 1 or strides[0] 

Author: Antonio Cuni 
Branch: extradoc
Changeset: r5533:5ff800bfd872
Date: 2015-04-17 00:40 +0200
http://bitbucket.org/pypy/extradoc/changeset/5ff800bfd872/

Log:	more slides

diff --git a/talk/pycon-italy-2015/talk.rst b/talk/pycon-italy-2015/talk.rst
--- a/talk/pycon-italy-2015/talk.rst
+++ b/talk/pycon-italy-2015/talk.rst
@@ -121,7 +121,102 @@
 
 .. image:: jit-overview3.pdf
    :scale: 50%
-          
+
+
+JIT overview
+-------------
+
+- Tracing JIT
+
+  * detect and compile "hot" loops
+
+  * (although not only loops)
+
+- **Specialization**
+
+- Precompute as much as possible
+
+- Constant propagation
+
+- Aggressive inlining
+
+
+Specialization (1)
+-------------------
+
+- ``obj.foo()``
+
+- which code is executed? (SIMPLIFIED)
+
+  * lookup ``foo`` in obj.__dict__
+
+  * lookup ``foo`` in obj.__class__
+
+  * lookup ``foo`` in obj.__bases__[0], etc.
+
+  * finally, execute ``foo``
+
+- without JIT, you need to do these steps again and again
+
+- Precompute the lookup?
+
+
+Specialization (2)
+--------------------
+
+- pretend and assume that ``obj.__class__`` IS constant
+
+  * "promotion"
+
+- guard
+
+  * check our assumption: if it's false, bail out
+
+- now we can directly jump to ``foo`` code
+
+  * ...unless ``foo`` is in ``obj.__dict__``: GUARD!
+
+  * ...unless ``foo.__class__.__dict__`` changed: GUARD!
+
+- Too many guard failures?
+
+  * Compile some more assembler!
+
+- guards are cheap
+
+  * out-of-line guards even more
+
+
+Specialization (3)
+---------------------
+
+- who decides what to promote/specialize for?
+
+  * we, the PyPy devs :)
+
+  * heuristics
+
+- instance attributes are never promoted
+
+- class attributes are promoted by default (with some exceptions)
+
+- module attributes (i.e., globals) as well
+
+- bytecode constants
+
+
+Specialization trade-offs
+--------------------------
+
+- Too much specialization
+
+  * guards fails often
+
+  * explosion of assembler
+
+- Not enough specialization
+
+  * inefficient code
 
 
 

From noreply at buildbot.pypy.org  Fri Apr 17 01:05:21 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 17 Apr 2015 01:05:21 +0200 (CEST)
Subject: [pypy-commit] cffi win32-callback: passes on cpython 2.7.9 win32,
 fails on win64 (from cpython issue #20160)
Message-ID: <20150416230521.22F761C1465@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: win32-callback
Changeset: r1738:a8893bfe2231
Date: 2015-04-17 02:05 +0300
http://bitbucket.org/cffi/cffi/changeset/a8893bfe2231/

Log:	passes on cpython 2.7.9 win32, fails on win64 (from cpython issue
	#20160)

diff --git a/testing/test_ownlib.py b/testing/test_ownlib.py
--- a/testing/test_ownlib.py
+++ b/testing/test_ownlib.py
@@ -91,6 +91,19 @@
     return ar;
 }
 
+typedef int (*callback_RECT)(RECT r);
+typedef int (*callback_POINT)(POINT r);
+
+EXPORT int Callback_RECT_Test( RECT r, callback_RECT cb)
+{
+    return cb(r);
+}
+
+EXPORT int Callback_POINT_Test( POINT p, callback_POINT cb)
+{
+    return cb(p);
+}
+
 EXPORT int my_array[7] = {0, 1, 2, 3, 4, 5, 6};
 """
 
@@ -282,3 +295,61 @@
             assert ret.right == ownlib.right
             assert ret.top == ownlib.top
             assert ret.bottom == ownlib.bottom
+
+    def test_struct_callback(self):
+        if self.module is None:
+            py.test.skip("fix the auto-generation of the tiny test lib")
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("""
+            typedef struct {
+                long x;
+                long y;
+            } POINT;
+
+            typedef struct {
+                long left;
+                long top;
+                long right;
+                long bottom;
+            } RECT;
+
+            typedef int (*callback_RECT)(RECT r);
+            typedef int (*callback_POINT)(POINT r);
+            
+            int Callback_RECT_Test( RECT r, callback_RECT cb);
+            int Callback_POINT_Test( POINT r, callback_POINT cb);
+            """)
+        ownlib = ffi.dlopen(self.module)
+
+        rect = ffi.new('RECT[1]')
+        pt = ffi.new('POINT[1]')
+        pt[0].x = 100
+        pt[0].y = 200
+        rect[0].left = 10
+        rect[0].right = 20
+        rect[0].top = 30
+        rect[0].bottom = 40
+        def check_rect(cb_rect):
+            if cb_rect.left != rect[0].left:
+                print 'got',cb_rect.left,'wanted',rect[0].left
+                return 0
+            if cb_rect.top != rect[0].top:
+                return 0
+            if cb_rect.right != rect[0].right:
+                return 0
+            if cb_rect.bottom != rect[0].bottom:
+                return 0
+            return 1
+        cb = ffi.callback('int(RECT)', check_rect)
+        result = ownlib.Callback_RECT_Test(rect[0], cb)
+        assert result == 1
+
+        def check_point(cb_point):
+            if cb_point.x != pt[0].x:
+                return 0
+            if cb_point.y != pt[0].y:
+                return 0
+            return 1
+        cb = ffi.callback('int(POINT)', check_point)
+        result = ownlib.Callback_POINT_Test(pt[0], cb)
+        assert result == 1

From noreply at buildbot.pypy.org  Fri Apr 17 01:57:22 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Fri, 17 Apr 2015 01:57:22 +0200 (CEST)
Subject: [pypy-commit] pypy object-dtype2: progress on translation
Message-ID: <20150416235722.D6F611C1027@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: object-dtype2
Changeset: r76806:82829532ae2d
Date: 2015-04-17 00:08 +0100
http://bitbucket.org/pypy/pypy/changeset/82829532ae2d/

Log:	progress on translation

diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1726,7 +1726,7 @@
     def for_computation(v):
         return v
 
-    @simple_binary_op
+    @raw_binary_op
     def eq(self, v1, v2):
         return self.space.eq_w(v1, v2)
 
@@ -1750,23 +1750,23 @@
             return True
         return False
 
-    @simple_binary_op
+    @raw_binary_op
     def logical_and(self, v1, v2):
         if self._obool(v1):
             return v2
         return v1
 
-    @simple_binary_op
+    @raw_binary_op
     def logical_or(self, v1, v2):
         if self._obool(v1):
             return v1
         return v2
 
-    @simple_unary_op
+    @raw_unary_op
     def logical_not(self, v):
         return not self._obool(v)
 
-    @simple_binary_op
+    @raw_binary_op
     def logical_xor(self, v1, v2):
         a = self._obool(v1)
         b = self._obool(v2)
@@ -1816,23 +1816,23 @@
     def square(self, v):
         return self.space.mul(v, v)
 
-    @simple_binary_op
+    @raw_binary_op
     def le(self, v1, v2):
         return self.space.le(v1, v2)
 
-    @simple_binary_op
+    @raw_binary_op
     def ge(self, v1, v2):
         return self.space.ge(v1, v2)
 
-    @simple_binary_op
+    @raw_binary_op
     def lt(self, v1, v2):
         return self.space.lt(v1, v2)
 
-    @simple_binary_op
+    @raw_binary_op
     def gt(self, v1, v2):
         return self.space.gt(v1, v2)
 
-    @simple_binary_op
+    @raw_binary_op
     def ne(self, v1, v2):
         return self.space.ne(v1, v2)
 
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -483,7 +483,7 @@
         else:
             out = w_out
             calc_dtype = out.get_dtype()
-        if self.comparison_func and not calc_dtype.is_object():
+        if self.comparison_func:
             res_dtype = get_dtype_cache(space).w_booldtype
         else:
             res_dtype = calc_dtype
@@ -497,13 +497,8 @@
                     out.set_scalar_value(arr)
                 else:
                     out.fill(space, arr)
-            elif isinstance(arr, res_dtype.itemtype.BoxType):
-                if res_dtype.is_object():
-                    out = arr.get_scalar_value()
-                else:
-                    out = arr
             else:
-                out = space.wrap(arr)
+                out = arr
             return out
         if isinstance(w_lhs, boxes.W_GenericBox):
             w_lhs = W_NDimArray.from_scalar(space, w_lhs)
@@ -652,7 +647,7 @@
             # from frompyfunc
             pass
         # mimic NpyIter_AdvancedNew with a nditer
-        w_itershape = space.newlist([space.wrap(i) for i in iter_shape]) 
+        w_itershape = space.newlist([space.wrap(i) for i in iter_shape])
         nd_it = W_NDIter(space, space.newlist(inargs + outargs), w_flags,
                       w_op_flags, w_op_dtypes, w_casting, w_op_axes,
                       w_itershape)
@@ -758,7 +753,7 @@
                 else:
                     raise oefmt(space.w_TypeError, "a type-string for %s " \
                         "requires 1 typecode or %d typecode(s) before and %d" \
-                        " after the -> sign, not '%s'", self.name, self.nin, 
+                        " after the -> sign, not '%s'", self.name, self.nin,
                         self.nout, type_tup)
             except KeyError:
                 raise oefmt(space.w_ValueError, "unknown typecode in" \
@@ -782,11 +777,11 @@
             for j in range(self.nargs):
                 if dtypes[j] is not None and dtypes[j] != _dtypes[i+j]:
                     allok = False
-            if allok:    
+            if allok:
                 break
         else:
             if len(self.funcs) > 1:
-                
+
                 dtypesstr = ''
                 for d in dtypes:
                     if d is None:
@@ -796,7 +791,7 @@
                 _dtypesstr = ','.join(['%s%s%s' % (d.byteorder, d.kind, d.elsize) \
                                 for d in _dtypes])
                 raise oefmt(space.w_TypeError,
-                     "input dtype [%s] did not match any known dtypes [%s] ", 
+                     "input dtype [%s] did not match any known dtypes [%s] ",
                      dtypesstr,_dtypesstr)
             i = 0
         # Fill in empty dtypes
@@ -816,7 +811,7 @@
             assert isinstance(curarg, W_NDimArray)
             if len(arg_shapes[i]) != curarg.ndims():
                 # reshape
-                
+
                 sz = product(curarg.get_shape()) * curarg.get_dtype().elsize
                 with curarg.implementation as storage:
                     inargs[i] = W_NDimArray.from_shape_and_storage(
@@ -874,7 +869,7 @@
                             "%s of gufunc was not specified",
                              self.name, name, _i, core_dim_index, self.signature)
                     target_dims.append(v)
-                arg_shapes.append(iter_shape + target_dims) 
+                arg_shapes.append(iter_shape + target_dims)
                 continue
             n = len(curarg.get_shape()) - num_dims
             if n < 0:
@@ -916,7 +911,7 @@
                     raise oefmt(space.w_ValueError, "%s: %s operand %d has a "
                         "mismatch in its core dimension %d, with gufunc "
                         "signature %s (expected %d, got %d)",
-                         self.name, name, _i, j, 
+                         self.name, name, _i, j,
                          self.signature, matched_dims[core_dim_index],
                          dims_to_match[core_dim_index])
             #print 'adding',iter_shape,'+',dims_to_match,'to arg_shapes'
@@ -1285,7 +1280,7 @@
      w_identity=None, name='', doc='', stack_inputs=False):
     ''' frompyfunc(func, nin, nout) #cpython numpy compatible
         frompyfunc(func, nin, nout, dtypes=None, signature='',
-                   identity=None, name='', doc='', 
+                   identity=None, name='', doc='',
                    stack_inputs=False)
 
     Takes an arbitrary Python function and returns a ufunc.
@@ -1304,7 +1299,7 @@
     dtypes: None or [dtype, ...] of the input, output args for each function,
          or 'match' to force output to exactly match input dtype
          Note that 'match' is a pypy-only extension to allow non-object
-         return dtypes      
+         return dtypes
     signature*: str, default=''
          The mapping of input args to output args, defining the
          inner-loop indexing. If it is empty, the func operates on scalars
@@ -1315,7 +1310,7 @@
     stack_inputs*: boolean, whether the function is of the form
             out = func(*in)  False
             or
-            func(*[in + out])    True 
+            func(*[in + out])    True
 
     only one of out_dtype or signature may be specified
 

From noreply at buildbot.pypy.org  Fri Apr 17 01:57:24 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Fri, 17 Apr 2015 01:57:24 +0200 (CEST)
Subject: [pypy-commit] pypy object-dtype2: fix translation (for real)
Message-ID: <20150416235724.07FB81C1027@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: object-dtype2
Changeset: r76807:711516940e44
Date: 2015-04-17 00:57 +0100
http://bitbucket.org/pypy/pypy/changeset/711516940e44/

Log:	fix translation (for real)

diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -890,10 +890,7 @@
                 "The truth value of an array with more than one element "
                 "is ambiguous. Use a.any() or a.all()"))
         iter, state = self.create_iter()
-        w_val = iter.getitem(state)
-        if self.get_dtype().is_object():
-            w_val = self.get_dtype().itemtype.unbox(w_val)
-        return space.wrap(space.is_true(w_val))
+        return space.wrap(space.is_true(iter.getitem(state)))
 
     def _binop_impl(ufunc_name):
         def impl(self, space, w_other, w_out=None):
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1718,6 +1718,10 @@
     def str_format(self, box):
         return self.space.str_w(self.space.repr(self.unbox(box)))
 
+    def runpack_str(self, space, s):
+        raise oefmt(space.w_NotImplementedError,
+                    "fromstring not implemented for object type")
+
     def to_builtin_type(self, space, box):
         assert isinstance(box, self.BoxType)
         return box.w_obj
@@ -1753,14 +1757,14 @@
     @raw_binary_op
     def logical_and(self, v1, v2):
         if self._obool(v1):
-            return v2
-        return v1
+            return self.space.bool_w(v2)
+        return self.space.bool_w(v1)
 
     @raw_binary_op
     def logical_or(self, v1, v2):
         if self._obool(v1):
-            return v1
-        return v2
+            return self.space.bool_w(v1)
+        return self.space.bool_w(v2)
 
     @raw_unary_op
     def logical_not(self, v):
@@ -1818,23 +1822,23 @@
 
     @raw_binary_op
     def le(self, v1, v2):
-        return self.space.le(v1, v2)
+        return self.space.bool_w(self.space.le(v1, v2))
 
     @raw_binary_op
     def ge(self, v1, v2):
-        return self.space.ge(v1, v2)
+        return self.space.bool_w(self.space.ge(v1, v2))
 
     @raw_binary_op
     def lt(self, v1, v2):
-        return self.space.lt(v1, v2)
+        return self.space.bool_w(self.space.lt(v1, v2))
 
     @raw_binary_op
     def gt(self, v1, v2):
-        return self.space.gt(v1, v2)
+        return self.space.bool_w(self.space.gt(v1, v2))
 
     @raw_binary_op
     def ne(self, v1, v2):
-        return self.space.ne(v1, v2)
+        return self.space.bool_w(self.space.ne(v1, v2))
 
 def add_attributeerr_op(cls, op):
     def func(self, *args):
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -1128,17 +1128,12 @@
         def impl(res_dtype, value):
             res = get_op(res_dtype)(value)
             if bool_result:
-                if res_dtype.is_object() and isinstance(res, W_Root):
-                    space = res_dtype.itemtype.space
-                    res = space.bool_w(res)
                 return dtype_cache.w_booldtype.box(res)
             return res
     elif nin == 2:
         def impl(res_dtype, lvalue, rvalue):
             res = get_op(res_dtype)(lvalue, rvalue)
             if comparison_func:
-                if res_dtype.is_object():
-                    return res
                 return dtype_cache.w_booldtype.box(res)
             return res
     return func_with_new_name(impl, ufunc_name)

From noreply at buildbot.pypy.org  Fri Apr 17 09:41:44 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 17 Apr 2015 09:41:44 +0200 (CEST)
Subject: [pypy-commit] pypy object-dtype2: restore a few methods
Message-ID: <20150417074144.377A01C0EDD@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: object-dtype2
Changeset: r76808:d9b54f7dd7e1
Date: 2015-04-17 10:41 +0300
http://bitbucket.org/pypy/pypy/changeset/d9b54f7dd7e1/

Log:	restore a few methods

diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -202,7 +202,7 @@
             return self
         elif isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool() \
                 and w_idx.ndims() > 0:
-            w_res = self.getitem_filter(space, w_idx)
+            return self.getitem_filter(space, w_idx)
         else:
             try:
                 return self.implementation.descr_getitem(space, self, w_idx)
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1857,10 +1857,11 @@
 def add_unary_op(cls, op, method):
     @simple_unary_op
     def func(self, w_v):
-        w_impl = getattr(w_v, method)
+        space = self.space
+        w_impl = space.lookup(w_v, method)
         if w_impl is None:
-            raise oefmt(self.space.w_AttributeError, 'unknown op "%s" on object' % op)
-        return w_impl(self.space)
+            raise oefmt(space.w_AttributeError, 'unknown op "%s" on object' % op)
+        return space.get_and_call_function(w_impl, w_v)
     func.__name__ = 'object_' + op
     setattr(cls, op, func)
 
@@ -1879,7 +1880,7 @@
     setattr(cls, op, func)
 
 for op in ('copysign', 'isfinite', 'isinf', 'isnan', 'logaddexp', 'logaddexp2',
-           'signbit', 'conj', 'rint'):
+           'signbit'):
     add_unsupported_op(ObjectType, op)
 for op in ('arctan2', 'arccos', 'arccosh', 'arcsin', 'arcsinh', 'arctan',
            'arctanh', 'ceil', 'floor', 'cos', 'sin', 'tan', 'cosh', 'sinh',
@@ -1888,6 +1889,8 @@
     add_attributeerr_op(ObjectType, op)
 for op in ('abs', 'neg', 'pos', 'invert'):
     add_space_unary_op(ObjectType, op)
+for op, method in (('conj', 'descr_conjugate'), ('rint', 'descr_rint')):
+    add_unary_op(ObjectType, op, method)
 for op in ('add', 'floordiv', 'div', 'mod', 'mul', 'sub', 'lshift', 'rshift'):
     add_space_binary_op(ObjectType, op)
 

From noreply at buildbot.pypy.org  Fri Apr 17 09:57:16 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 17 Apr 2015 09:57:16 +0200 (CEST)
Subject: [pypy-commit] pypy default: try to fix arm translation
Message-ID: <20150417075716.C1CDC1C1331@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r76809:14d685f7a343
Date: 2015-04-17 09:56 +0200
http://bitbucket.org/pypy/pypy/changeset/14d685f7a343/

Log:	try to fix arm translation

diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -883,7 +883,7 @@
         size = self.mc.get_relative_pos() 
         res = self.mc.materialize(self.cpu, allblocks,
                                    self.cpu.gc_ll_descr.gcrootmap)
-        self.cpu.asmmemmgr.register_codemap(
+        self.cpu.codemap.register_codemap(
             self.codemap.get_final_bytecode(res, size))
         return res
 

From noreply at buildbot.pypy.org  Fri Apr 17 10:01:10 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 17 Apr 2015 10:01:10 +0200 (CEST)
Subject: [pypy-commit] pypy default: Backed out changeset 14d685f7a343
Message-ID: <20150417080110.0641D1C1331@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r76810:90cb6e238388
Date: 2015-04-17 09:57 +0200
http://bitbucket.org/pypy/pypy/changeset/90cb6e238388/

Log:	Backed out changeset 14d685f7a343

diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -883,7 +883,7 @@
         size = self.mc.get_relative_pos() 
         res = self.mc.materialize(self.cpu, allblocks,
                                    self.cpu.gc_ll_descr.gcrootmap)
-        self.cpu.codemap.register_codemap(
+        self.cpu.asmmemmgr.register_codemap(
             self.codemap.get_final_bytecode(res, size))
         return res
 

From noreply at buildbot.pypy.org  Fri Apr 17 10:01:11 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 17 Apr 2015 10:01:11 +0200 (CEST)
Subject: [pypy-commit] pypy default: bacout
Message-ID: <20150417080111.9331F1C1331@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r76811:b8b236fde733
Date: 2015-04-17 09:59 +0200
http://bitbucket.org/pypy/pypy/changeset/b8b236fde733/

Log:	bacout

diff too long, truncating to 2000 out of 2560 lines

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -9,6 +9,7 @@
 from rpython.rlib.signature import signature
 from rpython.rlib.rarithmetic import r_uint, SHRT_MIN, SHRT_MAX, \
     INT_MIN, INT_MAX, UINT_MAX, USHRT_MAX
+from rpython.rlib.rweaklist import RWeakListMixin
 
 from pypy.interpreter.executioncontext import (ExecutionContext, ActionFlag,
     UserDelAction)
@@ -366,6 +367,10 @@
 
 # ____________________________________________________________
 
+class CodeObjWeakList(RWeakListMixin):
+    def __init__(self):
+        self.initialize()
+
 class ObjSpace(object):
     """Base class for the interpreter-level implementations of object spaces.
     http://pypy.readthedocs.org/en/latest/objspace.html"""
@@ -389,6 +394,7 @@
         self.check_signal_action = None   # changed by the signal module
         self.user_del_action = UserDelAction(self)
         self._code_of_sys_exc_info = None
+        self.all_code_objs = CodeObjWeakList()
 
         # can be overridden to a subclass
         self.initialize()
@@ -666,16 +672,16 @@
             assert ec is not None
             return ec
 
-    def register_code_callback(self, callback):
+    def register_code_object(self, pycode):
+        callback = self.getexecutioncontext().register_code_callback
+        if callback is not None:
+            callback(self, pycode)
+        self.all_code_objs.add_handle(pycode)
+
+    def set_code_callback(self, callback):
         ec = self.getexecutioncontext()
-        ec._code_callback = callback
-
-    def register_code_object(self, pycode):
-        ec = self.getexecutioncontext()
-        if ec._code_callback is None:
-            return
-        ec._code_callback(self, pycode)
-    
+        ec.register_code_callback = callback
+        
     def _freeze_(self):
         return True
 
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -2,7 +2,6 @@
 from pypy.interpreter.error import OperationError, get_cleared_operation_error
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib import jit
-from rpython.rlib.objectmodel import we_are_translated
 
 TICK_COUNTER_STEP = 100
 
@@ -34,16 +33,11 @@
         self.profilefunc = None
         self.w_profilefuncarg = None
         self.thread_disappeared = False   # might be set to True after os.fork()
-
+        self.register_code_callback = None
         if sys.maxint == 2147483647:
             self._code_unique_id = 0 # XXX this is wrong, it won't work on 32bit
         else:
-            if we_are_translated():
-                self._code_unique_id = 0x7000000000000000
-            else:
-                self._code_unique_id = 0x7700000000000000
-                # should be enough code objects
-        self._code_callback = None
+            self._code_unique_id = 0x7000000000000000
 
     @staticmethod
     def _mark_thread_disappeared(space):
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -129,7 +129,7 @@
 
         ec = self.space.getexecutioncontext()
         self._unique_id = ec._code_unique_id
-        ec._code_unique_id += 4 # so we have two bits that we can mark stuff
+        ec._code_unique_id += 2 # so we have one bit that we can mark stuff
         # with
 
     def _get_full_name(self):
diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -3,15 +3,13 @@
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rtyper.annlowlevel import cast_instance_to_gcref, cast_base_ptr_to_instance
 from rpython.rlib.objectmodel import we_are_translated
-from rpython.rlib import jit, rposix, rgc
-from rpython.rlib.rarithmetic import ovfcheck_float_to_int
+from rpython.rlib import jit, rposix, entrypoint
 from rpython.rtyper.tool import rffi_platform as platform
 from rpython.rlib.rstring import StringBuilder
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import oefmt, wrap_oserror, OperationError
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.pyframe import PyFrame
-from pypy.interpreter.pycode import PyCode
 
 ROOT = py.path.local(__file__).join('..')
 SRC = ROOT.join('src')
@@ -30,13 +28,14 @@
     libraries = ['unwind'],
     
     post_include_bits=["""
+        void* pypy_vmprof_get_virtual_ip(void*);
         void pypy_vmprof_init(void);
     """],
     
     separate_module_sources=["""
         void pypy_vmprof_init(void) {
             vmprof_set_mainloop(pypy_execute_frame_trampoline, 0,
-                                NULL);
+                                pypy_vmprof_get_virtual_ip);
         }
     """],
     )
@@ -57,7 +56,7 @@
 
 pypy_execute_frame_trampoline = rffi.llexternal(
     "pypy_execute_frame_trampoline",
-    [llmemory.GCREF, llmemory.GCREF, llmemory.GCREF, lltype.Signed],
+    [llmemory.GCREF, llmemory.GCREF, llmemory.GCREF],
     llmemory.GCREF,
     compilation_info=eci,
     _nowrapper=True, sandboxsafe=True,
@@ -97,15 +96,23 @@
             gc_frame = cast_instance_to_gcref(frame)
             gc_inputvalue = cast_instance_to_gcref(w_inputvalue)
             gc_operr = cast_instance_to_gcref(operr)
-            assert frame.pycode._unique_id & 3 == 0
-            unique_id = frame.pycode._unique_id | 1
-            gc_result = pypy_execute_frame_trampoline(gc_frame, gc_inputvalue,
-                                                      gc_operr, unique_id)
+            gc_result = pypy_execute_frame_trampoline(gc_frame, gc_inputvalue, gc_operr)
             return cast_base_ptr_to_instance(W_Root, gc_result)
         else:
             return original_execute_frame(frame, w_inputvalue, operr)
 
 
+ at entrypoint.entrypoint_lowlevel('main', [llmemory.GCREF],
+                                'pypy_vmprof_get_virtual_ip', True)
+def get_virtual_ip(gc_frame):
+    frame = cast_base_ptr_to_instance(PyFrame, gc_frame)
+    if jit._get_virtualizable_token(frame):
+        return rffi.cast(rffi.VOIDP, 0)
+    virtual_ip = do_get_virtual_ip(frame)
+    return rffi.cast(rffi.VOIDP, virtual_ip)
+
+def do_get_virtual_ip(frame):
+    return frame.pycode._unique_id
 
 def write_long_to_string_builder(l, b):
     if sys.maxint == 2147483647:
@@ -123,33 +130,31 @@
         b.append(chr((l >> 48) & 0xff))
         b.append(chr((l >> 56) & 0xff))
 
-def try_cast_to_pycode(gcref):
-    return rgc.try_cast_gcref_to_instance(PyCode, gcref)
-
-MAX_CODES = 1000
-
 class VMProf(object):
     def __init__(self):
         self.is_enabled = False
         self.ever_enabled = False
+        self.mapping_so_far = [] # stored mapping in between runs
         self.fileno = -1
-        self.current_codes = []
 
-    def enable(self, space, fileno, period_usec):
+    def enable(self, space, fileno, period):
         if self.is_enabled:
             raise oefmt(space.w_ValueError, "_vmprof already enabled")
         self.fileno = fileno
         self.is_enabled = True
-        self.write_header(fileno, period_usec)
+        self.write_header(fileno, period)
         if not self.ever_enabled:
             if we_are_translated():
                 pypy_vmprof_init()
             self.ever_enabled = True
-        self.gather_all_code_objs(space)
-        space.register_code_callback(vmprof_register_code)
+        for weakcode in space.all_code_objs.get_all_handles():
+            code = weakcode()
+            if code:
+                self.register_code(space, code)
+        space.set_code_callback(vmprof_register_code)
         if we_are_translated():
             # does not work untranslated
-            res = vmprof_enable(fileno, period_usec, 0,
+            res = vmprof_enable(fileno, period, 0,
                                 lltype.nullptr(rffi.CCHARP.TO), 0)
         else:
             res = 0
@@ -157,55 +162,42 @@
             raise wrap_oserror(space, OSError(rposix.get_saved_errno(),
                                               "_vmprof.enable"))
 
-    def gather_all_code_objs(self, space):
-        all_code_objs = rgc.do_get_objects(try_cast_to_pycode)
-        for code in all_code_objs:
-            self.register_code(space, code)
-
-    def write_header(self, fileno, period_usec):
-        assert period_usec > 0
+    def write_header(self, fileno, period):
+        if period == -1:
+            period_usec = 1000000 / 100 #  100hz
+        else:
+            period_usec = period
         b = StringBuilder()
         write_long_to_string_builder(0, b)
         write_long_to_string_builder(3, b)
         write_long_to_string_builder(0, b)
         write_long_to_string_builder(period_usec, b)
         write_long_to_string_builder(0, b)
-        b.append('\x04') # interp name
-        b.append(chr(len('pypy')))
-        b.append('pypy')
         os.write(fileno, b.build())
 
     def register_code(self, space, code):
         if self.fileno == -1:
             raise OperationError(space.w_RuntimeError,
                                  space.wrap("vmprof not running"))
-        self.current_codes.append(code)
-        if len(self.current_codes) >= MAX_CODES:
-            self._flush_codes(space)
-
-    def _flush_codes(self, space):
+        name = code._get_full_name()
         b = StringBuilder()
-        for code in self.current_codes:
-            name = code._get_full_name()
-            b.append('\x02')
-            write_long_to_string_builder(code._unique_id, b)
-            write_long_to_string_builder(len(name), b)
-            b.append(name)
+        b.append('\x02')
+        write_long_to_string_builder(code._unique_id, b)
+        write_long_to_string_builder(len(name), b)
+        b.append(name)
         os.write(self.fileno, b.build())
-        self.current_codes = []
 
     def disable(self, space):
         if not self.is_enabled:
             raise oefmt(space.w_ValueError, "_vmprof not enabled")
         self.is_enabled = False
-        space.register_code_callback(None)
-        self._flush_codes(space)
         self.fileno = -1
         if we_are_translated():
            # does not work untranslated
             res = vmprof_disable()
         else:
             res = 0
+        space.set_code_callback(None)
         if res == -1:
             raise wrap_oserror(space, OSError(rposix.get_saved_errno(),
                                               "_vmprof.disable"))
@@ -215,23 +207,13 @@
     mod_vmprof = space.getbuiltinmodule('_vmprof')
     assert isinstance(mod_vmprof, Module)
     mod_vmprof.vmprof.register_code(space, code)
-
- at unwrap_spec(fileno=int, period=float)
-def enable(space, fileno, period=0.01):   # default 100 Hz
+        
+ at unwrap_spec(fileno=int, period=int)
+def enable(space, fileno, period=-1):
     from pypy.module._vmprof import Module
     mod_vmprof = space.getbuiltinmodule('_vmprof')
     assert isinstance(mod_vmprof, Module)
-    #
-    try:
-        period_usec = ovfcheck_float_to_int(period * 1000000.0 + 0.5)
-        if period_usec <= 0 or period_usec >= 1e6:
-            # we don't want seconds here at all
-            raise ValueError
-    except (ValueError, OverflowError):
-        raise OperationError(space.w_ValueError,
-                             space.wrap("'period' too large or non positive"))
-    #
-    mod_vmprof.vmprof.enable(space, fileno, period_usec)
+    mod_vmprof.vmprof.enable(space, fileno, period)
 
 def disable(space):
     from pypy.module._vmprof import Module
diff --git a/pypy/module/_vmprof/src/fake_pypy_api.c b/pypy/module/_vmprof/src/fake_pypy_api.c
--- a/pypy/module/_vmprof/src/fake_pypy_api.c
+++ b/pypy/module/_vmprof/src/fake_pypy_api.c
@@ -1,15 +1,25 @@
+
+long pypy_jit_start_addr(void)
+{
+	return 3;
+}
+
+long pypy_jit_end_addr(void)
+{
+	return 3;
+}
 
 long pypy_jit_stack_depth_at_loc(long x)
 {
 	return 0;
 }
 
-void *pypy_find_codemap_at_addr(long x)
+long pypy_find_codemap_at_addr(long x)
 {
-	return (void *)0;
+	return 0;
 }
 
-long pypy_yield_codemap_at_addr(void *x, long y, long *a)
+long pypy_yield_codemap_at_addr(long x, long y, long *a)
 {
 	return 0;
 }
@@ -17,5 +27,3 @@
 void pypy_pyframe_execute_frame(void)
 {
 }
-
-volatile int pypy_codemap_currently_invalid = 0;
diff --git a/pypy/module/_vmprof/src/get_custom_offset.c b/pypy/module/_vmprof/src/get_custom_offset.c
--- a/pypy/module/_vmprof/src/get_custom_offset.c
+++ b/pypy/module/_vmprof/src/get_custom_offset.c
@@ -1,65 +1,46 @@
 
-extern volatile int pypy_codemap_currently_invalid;
-
-void *pypy_find_codemap_at_addr(long addr, long *start_addr);
-long pypy_yield_codemap_at_addr(void *codemap_raw, long addr,
-                                long *current_pos_addr);
-long pypy_jit_stack_depth_at_loc(long loc);
-
+long pypy_jit_start_addr();
+long pypy_jit_end_addr();
+long pypy_jit_stack_depth_at_loc(long);
+long pypy_find_codemap_at_addr(long);
+long pypy_yield_codemap_at_addr(long, long, long*);
 
 void vmprof_set_tramp_range(void* start, void* end)
 {
 }
 
-int custom_sanity_check()
-{
-    return !pypy_codemap_currently_invalid;
-}
+static ptrdiff_t vmprof_unw_get_custom_offset(void* ip, unw_cursor_t *cp) {
+	intptr_t ip_l = (intptr_t)ip;
 
-static ptrdiff_t vmprof_unw_get_custom_offset(void* ip, void *cp) {
-    intptr_t ip_l = (intptr_t)ip;
-    return pypy_jit_stack_depth_at_loc(ip_l);
+	if (ip_l < pypy_jit_start_addr() || ip_l > pypy_jit_end_addr()) {
+		return -1;
+	}
+	return (void*)pypy_jit_stack_depth_at_loc(ip_l);
 }
 
 static long vmprof_write_header_for_jit_addr(void **result, long n,
-                                             void *ip, int max_depth)
+											 void *ip, int max_depth)
 {
-    void *codemap;
-    long current_pos = 0;
-    intptr_t id;
-    long start_addr = 0;
-    intptr_t addr = (intptr_t)ip;
-    int start, k;
-    void *tmp;
+	long codemap_pos;
+	long current_pos = 0;
+	intptr_t id;
+	intptr_t addr = (intptr_t)ip;
 
-    codemap = pypy_find_codemap_at_addr(addr, &start_addr);
-    if (codemap == NULL)
-        // not a jit code at all
-        return n;
-
-    // modify the last entry to point to start address and not the random one
-    // in the middle
-    result[n - 1] = (void*)start_addr;
-    start = n;
-    while (n < max_depth) {
-        id = pypy_yield_codemap_at_addr(codemap, addr, ¤t_pos);
-        if (id == 0)
-            // finish
-            break;
-        result[n++] = (void *)id;
-    }
-    // we strip the topmost part - the reason is that it's either
-    // represented in the jitted caller or it's not jitted (we have the
-    // same function essentially twice
-    k = 0;
-    while (k < (n - start) / 2) {
-        tmp = result[start + k];
-        result[start + k] = result[n - k - 1];
-        result[n - k - 1] = tmp;
-        k++;
-    }
-    if (n != max_depth) {
-        n--;
-    }
-    return n;
+	if (addr < pypy_jit_start_addr() || addr > pypy_jit_end_addr()) {
+		return n;
+	}
+	codemap_pos = pypy_find_codemap_at_addr(addr);
+	if (codemap_pos == -1) {
+		return n;
+	}
+	while (1) {
+		id = pypy_yield_codemap_at_addr(codemap_pos, addr, ¤t_pos);
+		if (id == 0) {
+			return n;
+		}
+		result[n++] = id;
+		if (n >= max_depth) {
+			return n;
+		}
+	}
 }
diff --git a/pypy/module/_vmprof/src/trampoline.asmgcc.s b/pypy/module/_vmprof/src/trampoline.asmgcc.s
--- a/pypy/module/_vmprof/src/trampoline.asmgcc.s
+++ b/pypy/module/_vmprof/src/trampoline.asmgcc.s
@@ -6,10 +6,11 @@
 	.type	pypy_execute_frame_trampoline, @function
 pypy_execute_frame_trampoline:
 	.cfi_startproc
-	pushq	%rcx
+	pushq	%rdi
 	.cfi_def_cfa_offset 16
 	call pypy_pyframe_execute_frame at PLT
-	popq	%rcx
+	/* GCROOT 0(%rsp) */
+	popq	%rdi
 	.cfi_def_cfa_offset 8
 	ret
 	.cfi_endproc
diff --git a/pypy/module/_vmprof/src/trampoline.h b/pypy/module/_vmprof/src/trampoline.h
--- a/pypy/module/_vmprof/src/trampoline.h
+++ b/pypy/module/_vmprof/src/trampoline.h
@@ -1,1 +1,1 @@
-void* pypy_execute_frame_trampoline(void*, void*, void*, long);
+void* pypy_execute_frame_trampoline(void*, void*, void*);
diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c
--- a/pypy/module/_vmprof/src/vmprof.c
+++ b/pypy/module/_vmprof/src/vmprof.c
@@ -25,8 +25,6 @@
 #include 
 #include 
 #include 
-#include 
-#include 
 
 #define UNW_LOCAL_ONLY
 #include 
@@ -36,18 +34,12 @@
 #define _unused(x) ((void)x)
 
 #define MAX_FUNC_NAME 128
-#define MAX_STACK_DEPTH 1024
-#define BUFFER_SIZE 8192
+#define MAX_STACK_DEPTH 64
 
-
-static int profile_file = 0;
-static char profile_write_buffer[BUFFER_SIZE];
-static int profile_buffer_position = 0;
+static FILE* profile_file = NULL;
 void* vmprof_mainloop_func;
 static ptrdiff_t mainloop_sp_offset;
 static vmprof_get_virtual_ip_t mainloop_get_virtual_ip;
-static long last_period_usec = 0;
-static int atfork_hook_installed = 0;
 
 
 /* *************************************************************
@@ -59,33 +51,27 @@
 #define MARKER_VIRTUAL_IP '\x02'
 #define MARKER_TRAILER '\x03'
 
-static void prof_word(long x) {
-	((long*)(profile_write_buffer + profile_buffer_position))[0] = x;
-	profile_buffer_position += sizeof(long);
+static void prof_word(FILE* f, long x) {
+    fwrite(&x, sizeof(x), 1, f);
 }
 
-static void prof_header(long period_usec) {
-    // XXX never used here?
-    prof_word(0);
-    prof_word(3);
-    prof_word(0);
-    prof_word(period_usec);
-    prof_word(0);
-    write(profile_file, profile_write_buffer, profile_buffer_position);
-    profile_buffer_position = 0;
+static void prof_header(FILE* f, long period_usec) {
+    prof_word(f, 0);
+    prof_word(f, 3);
+    prof_word(f, 0);
+    prof_word(f, period_usec);
+    prof_word(f, 0);
 }
 
-static void prof_write_stacktrace(void** stack, int depth, int count) {
+static void prof_write_stacktrace(FILE* f, void** stack, int depth, int count) {
     int i;
 	char marker = MARKER_STACKTRACE;
 
-	profile_write_buffer[profile_buffer_position++] = MARKER_STACKTRACE;
-    prof_word(count);
-    prof_word(depth);
+	fwrite(&marker, 1, 1, f);
+    prof_word(f, count);
+    prof_word(f, depth);
     for(i=0; i= 0);
     _unused(ret);
-	int first_run = 1;
 
     while (n < max_depth) {
         if (unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *) &ip) < 0) {
@@ -192,21 +173,16 @@
           void **arg_ptr = (void**)arg_addr;
           // fprintf(stderr, "stacktrace mainloop: rsp %p   &f2 %p   offset %ld\n", 
           //         sp, arg_addr, mainloop_sp_offset);
-		  if (mainloop_get_virtual_ip) {
-			  ip = mainloop_get_virtual_ip(*arg_ptr);
-		  } else {
-			  ip = *arg_ptr;
-		  }
+          ip = mainloop_get_virtual_ip(*arg_ptr);
         }
 
         result[n++] = ip;
 		n = vmprof_write_header_for_jit_addr(result, n, ip, max_depth);
-        if (vmprof_unw_step(&cursor, first_run) <= 0) {
+        if (vmprof_unw_step(&cursor) <= 0) {
             break;
         }
-		first_run = 0;
     }
-    //--recursive;
+    --recursive;
     return n;
 }
 
@@ -217,12 +193,10 @@
 
 static void sigprof_handler(int sig_nr, siginfo_t* info, void *ucontext) {
     void* stack[MAX_STACK_DEPTH];
-    int saved_errno = errno;
     stack[0] = GetPC((ucontext_t*)ucontext);
     int depth = frame_forcer(get_stack_trace(stack+1, MAX_STACK_DEPTH-1, ucontext));
     depth++;  // To account for pc value in stack[0];
-    prof_write_stacktrace(stack, depth, 1);
-    errno = saved_errno;
+    prof_write_stacktrace(profile_file, stack, depth, 1);
 }
 
 /* *************************************************************
@@ -235,12 +209,14 @@
 	if ((fd = dup(fd)) == -1) {
 		return -1;
 	}
-	profile_buffer_position = 0;
-    profile_file = fd;
+    profile_file = fdopen(fd, "wb");
+	if (!profile_file) {
+		return -1;
+	}
 	if (write_header)
-		prof_header(period_usec);
+		prof_header(profile_file, period_usec);
 	if (s)
-		write(profile_file, s, slen);
+		fwrite(s, slen, 1, profile_file);
 	return 0;
 }
 
@@ -250,16 +226,16 @@
     char buf[BUFSIZ];
     size_t size;
 	int marker = MARKER_TRAILER;
-	write(profile_file, &marker, 1);
+	fwrite(&marker, 1, 1, profile_file);
 
     // copy /proc/PID/maps to the end of the profile file
     sprintf(buf, "/proc/%d/maps", getpid());
     src = fopen(buf, "r");    
     while ((size = fread(buf, 1, BUFSIZ, src))) {
-        write(profile_file, buf, size);
+        fwrite(buf, 1, size, profile_file);
     }
     fclose(src);
-    close(profile_file);
+    fclose(profile_file);
 	return 0;
 }
 
@@ -277,16 +253,15 @@
 }
 
 static int remove_sigprof_handler(void) {
-    sighandler_t res = signal(SIGPROF, SIG_DFL);
-	if (res == SIG_ERR) {
-		return -1;
-	}
+    //sighandler_t res = signal(SIGPROF, SIG_DFL);
+	//if (res == SIG_ERR) {
+	//	return -1;
+	//}
 	return 0;
 };
 
 static int install_sigprof_timer(long period_usec) {
     static struct itimerval timer;
-    last_period_usec = period_usec;
     timer.it_interval.tv_sec = 0;
     timer.it_interval.tv_usec = period_usec;
     timer.it_value = timer.it_interval;
@@ -298,45 +273,15 @@
 
 static int remove_sigprof_timer(void) {
     static struct itimerval timer;
-    last_period_usec = 0;
     timer.it_interval.tv_sec = 0;
     timer.it_interval.tv_usec = 0;
-    timer.it_value.tv_sec = 0;
-    timer.it_value.tv_usec = 0;
+    timer.it_value = timer.it_interval;
     if (setitimer(ITIMER_PROF, &timer, NULL) != 0) {
 		return -1;
     }
 	return 0;
 }
 
-static void atfork_disable_timer(void) {
-    remove_sigprof_timer();
-}
-
-static void atfork_enable_timer(void) {
-    install_sigprof_timer(last_period_usec);
-}
-
-static int install_pthread_atfork_hooks(void) {
-    /* this is needed to prevent the problems described there:
-         - http://code.google.com/p/gperftools/issues/detail?id=278
-         - http://lists.debian.org/debian-glibc/2010/03/msg00161.html
-
-        TL;DR: if the RSS of the process is large enough, the clone() syscall
-        will be interrupted by the SIGPROF before it can complete, then
-        retried, interrupted again and so on, in an endless loop.  The
-        solution is to disable the timer around the fork, and re-enable it
-        only inside the parent.
-    */
-    if (atfork_hook_installed)
-        return 0;
-    int ret = pthread_atfork(atfork_disable_timer, atfork_enable_timer, NULL);
-    if (ret != 0)
-        return -1;
-    atfork_hook_installed = 1;
-    return 0;
-}
-
 /* *************************************************************
  * public API
  * *************************************************************
@@ -352,7 +297,8 @@
 int vmprof_enable(int fd, long period_usec, int write_header, char *s,
 				  int slen)
 {
-    assert(period_usec > 0);
+    if (period_usec == -1)
+        period_usec = 1000000 / 100; /* 100hz */
     if (open_profile(fd, period_usec, write_header, s, slen) == -1) {
 		return -1;
 	}
@@ -362,9 +308,6 @@
     if (install_sigprof_timer(period_usec) == -1) {
 		return -1;
 	}
-    if (install_pthread_atfork_hooks() == -1) {
-        return -1;
-    }
 	return 0;
 }
 
@@ -382,7 +325,6 @@
 }
 
 void vmprof_register_virtual_function(const char* name, void* start, void* end) {
-	// XXX unused by pypy
     // for now *end is simply ignored
 	char buf[1024];
 	int lgt = strlen(name) + 2 * sizeof(long) + 1;
@@ -394,5 +336,5 @@
 	((void **)(((void*)buf) + 1))[0] = start;
 	((long *)(((void*)buf) + 1 + sizeof(long)))[0] = lgt - 2 * sizeof(long) - 1;
 	strncpy(buf + 2 * sizeof(long) + 1, name, 1024 - 2 * sizeof(long) - 1);
-	write(profile_file, buf, lgt);
+	fwrite(buf, lgt, 1, profile_file);
 }
diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py
--- a/pypy/module/_vmprof/test/test__vmprof.py
+++ b/pypy/module/_vmprof/test/test__vmprof.py
@@ -21,11 +21,6 @@
             i = 0
             count = 0
             i += 5 * WORD # header
-            assert s[i] == '\x04'
-            i += 1 # marker
-            assert s[i] == '\x04'
-            i += 1 # length
-            i += len('pypy')
             while i < len(s):
                 assert s[i] == '\x02'
                 i += 1
@@ -58,11 +53,3 @@
         assert "py:foo:" in s
         assert "py:foo2:" in s
         assert no_of_codes2 >= no_of_codes + 2 # some extra codes from tests
-
-    def test_enable_ovf(self):
-        import _vmprof
-        raises(ValueError, _vmprof.enable, 999, 0)
-        raises(ValueError, _vmprof.enable, 999, -2.5)
-        raises(ValueError, _vmprof.enable, 999, 1e300)
-        raises(ValueError, _vmprof.enable, 999, 1e300 * 1e300)
-        raises(ValueError, _vmprof.enable, 999, (1e300*1e300) / (1e300*1e300))
diff --git a/pypy/module/_vmprof/test/test_direct.py b/pypy/module/_vmprof/test/test_direct.py
deleted file mode 100644
--- a/pypy/module/_vmprof/test/test_direct.py
+++ /dev/null
@@ -1,66 +0,0 @@
-
-import cffi, py
-
-srcdir = py.path.local(__file__).join("..", "..", "src")
-
-ffi = cffi.FFI()
-ffi.cdef("""
-long vmprof_write_header_for_jit_addr(void **, long, void*, int);
-void *pypy_find_codemap_at_addr(long addr, long *start_addr);
-long pypy_yield_codemap_at_addr(void *codemap_raw, long addr,
-                                long *current_pos_addr);
-long buffer[];
-""")
-
-lib = ffi.verify("""
-volatile int pypy_codemap_currently_invalid = 0;
-
-long buffer[] = {0, 0, 0, 0, 0};
-
-
-
-void *pypy_find_codemap_at_addr(long addr, long *start_addr)
-{
-    return (void*)buffer;
-}
-
-long pypy_yield_codemap_at_addr(void *codemap_raw, long addr,
-                                long *current_pos_addr)
-{
-    long c = *current_pos_addr;
-    if (c >= 5)
-        return 0;
-    *current_pos_addr = c + 1;
-    return *((long*)codemap_raw + c);
-}
-
-
-""" + open(str(srcdir.join("get_custom_offset.c"))).read())
-
-class TestDirect(object):
-    def test_infrastructure(self):
-        cont = ffi.new("long[1]", [0])
-        buf = lib.pypy_find_codemap_at_addr(0, cont)
-        assert buf
-        cont[0] = 0
-        next_addr = lib.pypy_yield_codemap_at_addr(buf, 0, cont)
-        assert cont[0] == 1
-        assert not next_addr
-        lib.buffer[0] = 13
-        cont[0] = 0
-        next_addr = lib.pypy_yield_codemap_at_addr(buf, 0, cont)
-        assert int(ffi.cast("long", next_addr)) == 13
-
-    def test_write_header_for_jit_addr(self):
-        lib.buffer[0] = 4
-        lib.buffer[1] = 8
-        lib.buffer[2] = 12
-        lib.buffer[3] = 16
-        lib.buffer[4] = 0
-        buf = ffi.new("long[5]", [0] * 5)
-        result = ffi.cast("void**", buf)
-        res = lib.vmprof_write_header_for_jit_addr(result, 0, ffi.NULL, 100)
-        assert res == 3
-        assert buf[0] == 16
-        assert buf[1] == 12
-        assert buf[2] == 8
diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py
--- a/pypy/module/gc/referents.py
+++ b/pypy/module/gc/referents.py
@@ -44,6 +44,30 @@
     return OperationError(space.w_NotImplementedError,
                           space.wrap("operation not implemented by this GC"))
 
+# ____________________________________________________________
+
+def clear_gcflag_extra(fromlist):
+    pending = fromlist[:]
+    while pending:
+        gcref = pending.pop()
+        if rgc.get_gcflag_extra(gcref):
+            rgc.toggle_gcflag_extra(gcref)
+            pending.extend(rgc.get_rpy_referents(gcref))
+
+def do_get_objects():
+    roots = [gcref for gcref in rgc.get_rpy_roots() if gcref]
+    pending = roots[:]
+    result_w = []
+    while pending:
+        gcref = pending.pop()
+        if not rgc.get_gcflag_extra(gcref):
+            rgc.toggle_gcflag_extra(gcref)
+            w_obj = try_cast_gcref_to_w_root(gcref)
+            if w_obj is not None:
+                result_w.append(w_obj)
+            pending.extend(rgc.get_rpy_referents(gcref))
+    clear_gcflag_extra(roots)
+    return result_w
 
 # ____________________________________________________________
 
@@ -92,8 +116,8 @@
                 break
     # done.  Clear flags carefully
     rgc.toggle_gcflag_extra(gcarg)
-    rgc.clear_gcflag_extra(roots)
-    rgc.clear_gcflag_extra([gcarg])
+    clear_gcflag_extra(roots)
+    clear_gcflag_extra([gcarg])
     return result_w
 
 # ____________________________________________________________
@@ -165,7 +189,8 @@
     """Return a list of all app-level objects."""
     if not rgc.has_gcflag_extra():
         raise missing_operation(space)
-    result_w = rgc.do_get_objects(try_cast_gcref_to_w_root)
+    result_w = do_get_objects()
+    rgc.assert_no_more_gcflags()
     return space.newlist(result_w)
 
 def get_referents(space, args_w):
diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py
--- a/pypy/module/pypyjit/interp_resop.py
+++ b/pypy/module/pypyjit/interp_resop.py
@@ -105,7 +105,7 @@
             ofs = ops_offset.get(op, 0)
         if op.opnum == rop.DEBUG_MERGE_POINT:
             jd_sd = jitdrivers_sd[op.getarg(0).getint()]
-            greenkey = op.getarglist()[4:]
+            greenkey = op.getarglist()[3:]
             repr = jd_sd.warmstate.get_location_str(greenkey)
             w_greenkey = wrap_greenkey(space, jd_sd.jitdriver, greenkey, repr)
             l_w.append(DebugMergePoint(space, jit_hooks._cast_to_gcref(op),
diff --git a/pypy/module/pypyjit/test/test_jit_hook.py b/pypy/module/pypyjit/test/test_jit_hook.py
--- a/pypy/module/pypyjit/test/test_jit_hook.py
+++ b/pypy/module/pypyjit/test/test_jit_hook.py
@@ -55,7 +55,7 @@
         oplist = parse("""
         [i1, i2, p2]
         i3 = int_add(i1, i2)
-        debug_merge_point(0, 0, 0, 0, 0, 0, ConstPtr(ptr0))
+        debug_merge_point(0, 0, 0, 0, 0, ConstPtr(ptr0))
         guard_nonnull(p2) []
         guard_true(i3) []
         """, namespace={'ptr0': code_gcref}).operations
diff --git a/rpython/bin/rpython-vmprof b/rpython/bin/rpython-vmprof
deleted file mode 100755
--- a/rpython/bin/rpython-vmprof
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/usr/bin/env pypy
-
-"""RPython translation usage:
-
-rpython  target 
-
-run with --help for more information
-"""
-
-import sys, os
-sys.path.insert(0, os.path.dirname(os.path.dirname(
-                       os.path.dirname(os.path.realpath(__file__)))))
-from rpython.translator.goal.translate import main
-
-# no implicit targets
-if len(sys.argv) == 1:
-    print __doc__
-    sys.exit(1)
-
-import _vmprof, subprocess
-x = subprocess.Popen('gzip > vmprof.log.gz', shell=True, stdin=subprocess.PIPE)
-_vmprof.enable(x.stdin.fileno(), 0.001)
-try:
-    main()
-finally:
-    _vmprof.disable()
-    x.stdin.close()
-    x.wait()
diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -102,7 +102,7 @@
         self.store_reg(mc, r.r0, r.fp, ofs)
         mc.MOV_rr(r.r0.value, r.fp.value)
         self.gen_func_epilog(mc)
-        rawstart = mc.materialize(self.cpu, [])
+        rawstart = mc.materialize(self.cpu.asmmemmgr, [])
         self.propagate_exception_path = rawstart
 
     def _store_and_reset_exception(self, mc, excvalloc=None, exctploc=None,
@@ -198,7 +198,7 @@
         mc.ADD_ri(r.sp.value, r.sp.value, (len(r.argument_regs) + 2) * WORD)
         mc.B(self.propagate_exception_path)
         #
-        rawstart = mc.materialize(self.cpu, [])
+        rawstart = mc.materialize(self.cpu.asmmemmgr, [])
         self.stack_check_slowpath = rawstart
 
     def _build_wb_slowpath(self, withcards, withfloats=False, for_frame=False):
@@ -255,7 +255,7 @@
         #
         mc.POP([r.ip.value, r.pc.value])
         #
-        rawstart = mc.materialize(self.cpu, [])
+        rawstart = mc.materialize(self.cpu.asmmemmgr, [])
         if for_frame:
             self.wb_slowpath[4] = rawstart
         else:
@@ -276,7 +276,7 @@
                                       callee_only)
         # return
         mc.POP([r.ip.value, r.pc.value])
-        return mc.materialize(self.cpu, [])
+        return mc.materialize(self.cpu.asmmemmgr, [])
 
     def _build_malloc_slowpath(self, kind):
         """ While arriving on slowpath, we have a gcpattern on stack 0.
@@ -352,7 +352,7 @@
         mc.POP([r.ip.value, r.pc.value])
 
         #
-        rawstart = mc.materialize(self.cpu, [])
+        rawstart = mc.materialize(self.cpu.asmmemmgr, [])
         return rawstart
 
     def _reload_frame_if_necessary(self, mc):
@@ -473,7 +473,7 @@
         mc.MOV_rr(r.r0.value, r.fp.value)
         #
         self.gen_func_epilog(mc)
-        rawstart = mc.materialize(self.cpu, [])
+        rawstart = mc.materialize(self.cpu.asmmemmgr, [])
         self.failure_recovery_code[exc + 2 * withfloats] = rawstart
 
     def generate_quick_failure(self, guardtok):
@@ -851,7 +851,7 @@
         # restore registers
         self._pop_all_regs_from_jitframe(mc, [], self.cpu.supports_floats)
         mc.POP([r.ip.value, r.pc.value])  # return
-        self._frame_realloc_slowpath = mc.materialize(self.cpu, [])
+        self._frame_realloc_slowpath = mc.materialize(self.cpu.asmmemmgr, [])
 
     def _load_shadowstack_top(self, mc, reg, gcrootmap):
         rst = gcrootmap.get_root_stack_top_addr()
@@ -881,7 +881,7 @@
         self.datablockwrapper = None
         allblocks = self.get_asmmemmgr_blocks(looptoken)
         size = self.mc.get_relative_pos() 
-        res = self.mc.materialize(self.cpu, allblocks,
+        res = self.mc.materialize(self.cpu.asmmemmgr, allblocks,
                                    self.cpu.gc_ll_descr.gcrootmap)
         self.cpu.asmmemmgr.register_codemap(
             self.codemap.get_final_bytecode(res, size))
diff --git a/rpython/jit/backend/arm/runner.py b/rpython/jit/backend/arm/runner.py
--- a/rpython/jit/backend/arm/runner.py
+++ b/rpython/jit/backend/arm/runner.py
@@ -50,7 +50,6 @@
     def setup_once(self):
         self.cpuinfo.arch_version = detect_arch_version()
         self.cpuinfo.hf_abi = detect_hardfloat()
-        self.codemap.setup()
         self.assembler.setup_once()
 
     def finish_once(self):
diff --git a/rpython/jit/backend/arm/test/support.py b/rpython/jit/backend/arm/test/support.py
--- a/rpython/jit/backend/arm/test/support.py
+++ b/rpython/jit/backend/arm/test/support.py
@@ -24,7 +24,7 @@
 
 def run_asm(asm):
     BOOTSTRAP_TP = lltype.FuncType([], lltype.Signed)
-    addr = asm.mc.materialize(asm.cpu, [], None)
+    addr = asm.mc.materialize(asm.cpu.asmmemmgr, [], None)
     assert addr % 8 == 0
     func = rffi.cast(lltype.Ptr(BOOTSTRAP_TP), addr)
     asm.mc._dump_trace(addr, 'test.asm')
diff --git a/rpython/jit/backend/arm/test/test_calling_convention.py b/rpython/jit/backend/arm/test/test_calling_convention.py
--- a/rpython/jit/backend/arm/test/test_calling_convention.py
+++ b/rpython/jit/backend/arm/test/test_calling_convention.py
@@ -29,7 +29,7 @@
         mc = InstrBuilder()
         mc.MOV_rr(r.r0.value, r.sp.value)
         mc.MOV_rr(r.pc.value, r.lr.value)
-        return mc.materialize(self.cpu, [])
+        return mc.materialize(self.cpu.asmmemmgr, [])
 
     def get_alignment_requirements(self):
         return 8
diff --git a/rpython/jit/backend/llsupport/asmmemmgr.py b/rpython/jit/backend/llsupport/asmmemmgr.py
--- a/rpython/jit/backend/llsupport/asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/asmmemmgr.py
@@ -5,6 +5,9 @@
 from rpython.rlib.debug import debug_start, debug_print, debug_stop
 from rpython.rlib.debug import have_debug_prints
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rlib.rbisect import bisect, bisect_tuple
+
+_memmngr = None # global reference so we can use @entrypoint :/
 
 
 class AsmMemoryManager(object):
@@ -24,6 +27,12 @@
         self.free_blocks = {}      # map {start: stop}
         self.free_blocks_end = {}  # map {stop: start}
         self.blocks_by_size = [[] for i in range(self.num_indices)]
+        # two lists of jit addresses (sorted) and the corresponding stack
+        # depths
+        self.jit_addr_map = []
+        self.jit_frame_depth_map = []
+        self.jit_codemap = []
+        # see codemap.py
 
     def malloc(self, minsize, maxsize):
         """Allocate executable memory, between minsize and maxsize bytes,
@@ -45,6 +54,13 @@
         if r_uint is not None:
             self.total_mallocs -= r_uint(stop - start)
         self._add_free_block(start, stop)
+        # fix up jit_addr_map
+        jit_adr_start = bisect(self.jit_addr_map, start)
+        jit_adr_stop = bisect(self.jit_addr_map, stop)
+        self.jit_addr_map = (self.jit_addr_map[:jit_adr_start] +
+                             self.jit_addr_map[jit_adr_stop:])
+        self.jit_frame_depth_map = (self.jit_frame_depth_map[:jit_adr_start] +
+                                    self.jit_frame_depth_map[jit_adr_stop:])
 
     def open_malloc(self, minsize):
         """Allocate at least minsize bytes.  Returns (start, stop)."""
@@ -151,6 +167,35 @@
         del self.free_blocks_end[stop]
         return (start, stop)
 
+    def register_frame_depth_map(self, rawstart, frame_positions,
+                                 frame_assignments):
+        if not frame_positions:
+            return
+        if not self.jit_addr_map or rawstart > self.jit_addr_map[-1]:
+            start = len(self.jit_addr_map)
+            self.jit_addr_map += [0] * len(frame_positions)
+            self.jit_frame_depth_map += [0] * len(frame_positions)
+        else:
+            start = bisect(self.jit_addr_map, rawstart)
+            self.jit_addr_map = (self.jit_addr_map[:start] +
+                                 [0] * len(frame_positions) +
+                                 self.jit_addr_map[start:])
+            self.jit_frame_depth_map = (self.jit_frame_depth_map[:start] +
+                                 [0] * len(frame_positions) +
+                                 self.jit_frame_depth_map[start:])
+        for i, pos in enumerate(frame_positions):
+            self.jit_addr_map[i + start] = pos + rawstart
+            self.jit_frame_depth_map[i + start] = frame_assignments[i]
+
+    def register_codemap(self, codemap):
+        start = codemap[0]
+        pos = bisect_tuple(self.jit_codemap, start)
+        if pos == len(self.jit_codemap): # common case
+            self.jit_codemap.append(codemap)
+        else:
+            self.jit_codemap = (self.jit_codemap[:pos] + [codemap] +
+                                self.jit_codemap[pos:])
+
     def _delete(self):
         "NOT_RPYTHON"
         if self._allocated:
@@ -306,11 +351,11 @@
             #
         debug_stop(logname)
 
-    def materialize(self, cpu, allblocks, gcrootmap=None):
+    def materialize(self, asmmemmgr, allblocks, gcrootmap=None):
         size = self.get_relative_pos()
         align = self.ALIGN_MATERIALIZE
         size += align - 1
-        malloced = cpu.asmmemmgr.malloc(size, size)
+        malloced = asmmemmgr.malloc(size, size)
         allblocks.append(malloced)
         rawstart = malloced[0]
         rawstart = (rawstart + align - 1) & (-align)
@@ -319,9 +364,8 @@
             assert gcrootmap is not None
             for pos, mark in self.gcroot_markers:
                 gcrootmap.register_asm_addr(rawstart + pos, mark)
-        cpu.codemap.register_frame_depth_map(rawstart, rawstart + size,
-                                             self.frame_positions,
-                                             self.frame_assignments)
+        asmmemmgr.register_frame_depth_map(rawstart, self.frame_positions,
+                                           self.frame_assignments)
         self.frame_positions = None
         self.frame_assignments = None
         return rawstart
diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -130,7 +130,7 @@
         self.gcmap_for_finish[0] = r_uint(1)
 
     def setup(self, looptoken):
-        self.codemap_builder = CodemapBuilder()
+        self.codemap = CodemapBuilder()
         self._finish_gcmap = lltype.nullptr(jitframe.GCMAP)
 
     def set_debug(self, v):
@@ -200,9 +200,7 @@
         return fail_descr, target
 
     def debug_merge_point(self, op):
-        self.codemap_builder.debug_merge_point(op.getarg(1).getint(),
-                                               op.getarg(3).getint(),
-                                               self.mc.get_relative_pos())
+        self.codemap.debug_merge_point(op, self.mc.get_relative_pos())
 
     def call_assembler(self, op, guard_op, argloc, vloc, result_loc, tmploc):
         self._store_force_index(guard_op)
diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py
--- a/rpython/jit/backend/llsupport/codemap.py
+++ b/rpython/jit/backend/llsupport/codemap.py
@@ -9,129 +9,79 @@
 
 """
 
-import os
 from rpython.rlib import rgc
-from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.rlib.entrypoint import jit_entrypoint
-from rpython.rlib.rbisect import bisect_right, bisect_right_addr
-from rpython.rlib.rbisect import bisect_left, bisect_left_addr
+from rpython.jit.backend.llsupport import asmmemmgr
+from rpython.rlib.rbisect import bisect, bisect_tuple
 from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.translator import cdir
 
+ at jit_entrypoint([lltype.Signed], lltype.Signed,
+                c_name='pypy_jit_stack_depth_at_loc')
+ at rgc.no_collect
+def stack_depth_at_loc(loc):
+    _memmngr = asmmemmgr._memmngr
 
-INT_LIST_PTR = rffi.CArrayPtr(lltype.Signed)
+    pos = bisect(_memmngr.jit_addr_map, loc)
+    if pos == 0 or pos == len(_memmngr.jit_addr_map):
+        return -1
+    return _memmngr.jit_frame_depth_map[pos-1]
 
+ at jit_entrypoint([], lltype.Signed, c_name='pypy_jit_start_addr')
+def jit_start_addr():
+    _memmngr = asmmemmgr._memmngr
 
-srcdir = os.path.join(os.path.dirname(__file__), 'src')
+    return _memmngr.jit_addr_map[0]
 
-eci = ExternalCompilationInfo(post_include_bits=["""
-#include 
-RPY_EXTERN long pypy_jit_codemap_add(uintptr_t addr,
-                                     unsigned int machine_code_size,
-                                     long *bytecode_info,
-                                     unsigned int bytecode_info_size);
-RPY_EXTERN long *pypy_jit_codemap_del(uintptr_t addr);
-RPY_EXTERN uintptr_t pypy_jit_codemap_firstkey(void);
-RPY_EXTERN void *pypy_find_codemap_at_addr(long addr, long* start_addr);
-RPY_EXTERN long pypy_yield_codemap_at_addr(void *codemap_raw, long addr,
-                                           long *current_pos_addr);
+ at jit_entrypoint([], lltype.Signed, c_name='pypy_jit_end_addr')
+def jit_end_addr():
+    _memmngr = asmmemmgr._memmngr
 
-RPY_EXTERN long pypy_jit_depthmap_add(uintptr_t addr, unsigned int size,
-                                      unsigned int stackdepth);
-RPY_EXTERN void pypy_jit_depthmap_clear(uintptr_t addr, unsigned int size);
+    return _memmngr.jit_addr_map[-1]
 
-"""], separate_module_sources=[
-    open(os.path.join(srcdir, 'skiplist.c'), 'r').read() +
-    open(os.path.join(srcdir, 'codemap.c'), 'r').read()
-], include_dirs=[cdir])
+ at jit_entrypoint([lltype.Signed], lltype.Signed,
+                c_name='pypy_find_codemap_at_addr')
+def find_codemap_at_addr(addr):
+    _memmngr = asmmemmgr._memmngr
 
-def llexternal(name, args, res):
-    return rffi.llexternal(name, args, res, compilation_info=eci,
-                           releasegil=False)
+    res = bisect_tuple(_memmngr.jit_codemap, addr) - 1
+    if res == len(_memmngr.jit_codemap):
+        return -1
+    return res
 
-pypy_jit_codemap_add = llexternal('pypy_jit_codemap_add',
-                                  [lltype.Signed, lltype.Signed,
-                                   INT_LIST_PTR, lltype.Signed],
-                                  lltype.Signed)
-pypy_jit_codemap_del = llexternal('pypy_jit_codemap_del',
-                                  [lltype.Signed], INT_LIST_PTR)
-pypy_jit_codemap_firstkey = llexternal('pypy_jit_codemap_firstkey',
-                                       [], lltype.Signed)
+ at jit_entrypoint([lltype.Signed, lltype.Signed,
+                 rffi.CArrayPtr(lltype.Signed)], lltype.Signed,
+                 c_name='pypy_yield_codemap_at_addr')
+def yield_bytecode_at_addr(codemap_no, addr, current_pos_addr):
+    """ will return consecutive unique_ids from codemap, starting from position
+    `pos` until addr
+    """
+    _memmngr = asmmemmgr._memmngr
 
-pypy_jit_depthmap_add = llexternal('pypy_jit_depthmap_add',
-                                   [lltype.Signed, lltype.Signed,
-                                    lltype.Signed], lltype.Signed)
-pypy_jit_depthmap_clear = llexternal('pypy_jit_depthmap_clear',
-                                     [lltype.Signed, lltype.Signed],
-                                     lltype.Void)
-
-stack_depth_at_loc = llexternal('pypy_jit_stack_depth_at_loc',
-                                [lltype.Signed], lltype.Signed)
-find_codemap_at_addr = llexternal('pypy_find_codemap_at_addr',
-                                  [lltype.Signed, rffi.CArrayPtr(lltype.Signed)], lltype.Signed)
-yield_bytecode_at_addr = llexternal('pypy_yield_codemap_at_addr',
-                                    [lltype.Signed, lltype.Signed,
-                                     rffi.CArrayPtr(lltype.Signed)],
-                                     lltype.Signed)
-
-
-class CodemapStorage(object):
-    """ An immortal wrapper around underlaying jit codemap data
-    """
-    def setup(self):
-        if not we_are_translated():
-             # in case someone failed to call free(), in tests only anyway
-             self.free()
-
-    def free(self):
-        while True:
-            key = pypy_jit_codemap_firstkey()
-            if not key:
-                break
-            items = pypy_jit_codemap_del(key)
-            lltype.free(items, flavor='raw', track_allocation=False)
-
-    def free_asm_block(self, start, stop):
-        items = pypy_jit_codemap_del(start)
-        if items:
-            lltype.free(items, flavor='raw', track_allocation=False)
-        pypy_jit_depthmap_clear(start, stop - start)
-
-    def register_frame_depth_map(self, rawstart, rawstop, frame_positions,
-                                 frame_assignments):
-        if not frame_positions:
-            return
-        assert len(frame_positions) == len(frame_assignments)
-        for i in range(len(frame_positions)-1, -1, -1):
-            pos = rawstart + frame_positions[i]
-            length = rawstop - pos
-            if length > 0:
-                #print "ADD:", pos, length, frame_assignments[i]
-                pypy_jit_depthmap_add(pos, length, frame_assignments[i])
-            rawstop = pos
-
-    def register_codemap(self, (start, size, l)):
-        items = lltype.malloc(INT_LIST_PTR.TO, len(l), flavor='raw',
-                              track_allocation=False)
-        for i in range(len(l)):
-            items[i] = l[i]
-        if pypy_jit_codemap_add(start, size, items, len(l)) < 0:
-            lltype.free(items, flavor='raw', track_allocation=False)
-
-    def finish_once(self):
-        self.free()
+    codemap = _memmngr.jit_codemap[codemap_no]
+    current_pos = current_pos_addr[0]
+    start_addr = codemap[0]
+    rel_addr = addr - start_addr
+    while True:
+        if current_pos >= len(codemap[2]):
+            return 0
+        next_start = codemap[2][current_pos + 1]
+        if next_start > rel_addr:
+            return 0
+        next_stop = codemap[2][current_pos + 2]
+        if next_stop > rel_addr:
+            current_pos_addr[0] = current_pos + 4
+            return codemap[2][current_pos]
+        # we need to skip potentially more than one
+        current_pos = codemap[2][current_pos + 3]
 
 def unpack_traceback(addr):
-    codemap_raw = find_codemap_at_addr(addr,
-                                lltype.nullptr(rffi.CArray(lltype.Signed)))
-    if not codemap_raw:
-        return [] # no codemap for that position
+    codemap_pos = find_codemap_at_addr(addr)
+    assert codemap_pos >= 0
     storage = lltype.malloc(rffi.CArray(lltype.Signed), 1, flavor='raw')
     storage[0] = 0
     res = []
     while True:
-        item = yield_bytecode_at_addr(codemap_raw, addr, storage)
+        item = yield_bytecode_at_addr(codemap_pos, addr, storage)
         if item == 0:
             break
         res.append(item)
@@ -145,18 +95,14 @@
         self.patch_position = []
         self.last_call_depth = -1
 
-    def debug_merge_point(self, call_depth, unique_id, pos):
+    def debug_merge_point(self, op, pos):
+        call_depth = op.getarg(1).getint()
         if call_depth != self.last_call_depth:
+            unique_id = op.getarg(3).getint()
             if unique_id == 0: # uninteresting case
                 return
             assert unique_id & 1 == 0
             if call_depth > self.last_call_depth:
-                assert call_depth == self.last_call_depth + 1
-                # ^^^ It should never be the case that we see
-                # debug_merge_points that suddenly go more than *one*
-                # call deeper than the previous one (unless we're at
-                # the start of a bridge, handled by
-                # inherit_code_from_position()).
                 self.l.append(unique_id)
                 self.l.append(pos) # <- this is a relative pos
                 self.patch_position.append(len(self.l))
@@ -193,3 +139,4 @@
             item = self.l[i * 4 + 3] # end in l
             assert item > 0
         return (addr, size, self.l) # XXX compact self.l
+
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -16,7 +16,7 @@
     FieldDescr, ArrayDescr, CallDescr, InteriorFieldDescr,
     FLAG_POINTER, FLAG_FLOAT)
 from rpython.jit.backend.llsupport.memcpy import memset_fn
-from rpython.jit.backend.llsupport import asmmemmgr, codemap
+from rpython.jit.backend.llsupport import asmmemmgr
 from rpython.rlib.unroll import unrolling_iterable
 
 
@@ -49,7 +49,7 @@
         else:
             self._setup_exception_handling_untranslated()
         self.asmmemmgr = asmmemmgr.AsmMemoryManager()
-        self.codemap = codemap.CodemapStorage()
+        asmmemmgr._memmngr = self.asmmemmgr
         self._setup_frame_realloc(translate_support_code)
         ad = self.gc_ll_descr.getframedescrs(self).arraydescr
         self.signedarraydescr = ad
@@ -79,9 +79,6 @@
     def setup(self):
         pass
 
-    def finish_once(self):
-        self.codemap.finish_once()
-
     def _setup_frame_realloc(self, translate_support_code):
         FUNC_TP = lltype.Ptr(lltype.FuncType([llmemory.GCREF, lltype.Signed],
                                              llmemory.GCREF))
@@ -216,7 +213,6 @@
             for rawstart, rawstop in blocks:
                 self.gc_ll_descr.freeing_block(rawstart, rawstop)
                 self.asmmemmgr.free(rawstart, rawstop)
-                self.codemap.free_asm_block(rawstart, rawstop)
 
     def force(self, addr_of_force_token):
         frame = rffi.cast(jitframe.JITFRAMEPTR, addr_of_force_token)
diff --git a/rpython/jit/backend/llsupport/src/codemap.c b/rpython/jit/backend/llsupport/src/codemap.c
deleted file mode 100644
--- a/rpython/jit/backend/llsupport/src/codemap.c
+++ /dev/null
@@ -1,204 +0,0 @@
-#include "src/precommondefs.h"
-
-#ifndef HAS_SKIPLIST
-# error "skiplist.c needs to be included before"
-#endif
-
-volatile int pypy_codemap_currently_invalid = 0;
-
-void pypy_codemap_invalid_set(int value)
-{
-    if (value)
-        __sync_lock_test_and_set(&pypy_codemap_currently_invalid, 1);
-    else
-        __sync_lock_release(&pypy_codemap_currently_invalid);
-}
-
-
-/************************************************************/
-/***  codemap storage                                     ***/
-/************************************************************/
-
-typedef struct {
-    unsigned int machine_code_size;
-    unsigned int bytecode_info_size;
-    long *bytecode_info;
-} codemap_data_t;
-
-static skipnode_t jit_codemap_head;
-
-/*** interface used from codemap.py ***/
-
-RPY_EXTERN
-long pypy_jit_codemap_add(uintptr_t addr, unsigned int machine_code_size,
-                          long *bytecode_info, unsigned int bytecode_info_size)
-{
-    skipnode_t *new = skiplist_malloc(sizeof(codemap_data_t));
-    codemap_data_t *data;
-    if (new == NULL)
-        return -1;   /* too bad */
-
-    new->key = addr;
-    data = (codemap_data_t *)new->data;
-    data->machine_code_size = machine_code_size;
-    data->bytecode_info = bytecode_info;
-    data->bytecode_info_size = bytecode_info_size;
-
-    pypy_codemap_invalid_set(1);
-    skiplist_insert(&jit_codemap_head, new);
-    pypy_codemap_invalid_set(0);
-    return 0;
-}
-
-RPY_EXTERN
-long *pypy_jit_codemap_del(uintptr_t addr)
-{
-    long *result;
-    skipnode_t *node;
-
-    pypy_codemap_invalid_set(1);
-    node = skiplist_remove(&jit_codemap_head, addr);
-    pypy_codemap_invalid_set(0);
-
-    if (node == NULL)
-        return NULL;
-    result = ((codemap_data_t *)node->data)->bytecode_info;
-    free(node);
-    return result;
-}
-
-RPY_EXTERN
-uintptr_t pypy_jit_codemap_firstkey(void)
-{
-    return skiplist_firstkey(&jit_codemap_head);
-}
-
-/*** interface used from pypy/module/_vmprof ***/
-
-RPY_EXTERN
-void *pypy_find_codemap_at_addr(long addr, long* start_addr)
-{
-    skipnode_t *codemap = skiplist_search(&jit_codemap_head, addr);
-    codemap_data_t *data;
-    uintptr_t rel_addr;
-
-    if (codemap == &jit_codemap_head) {
-        if (start_addr)
-            *start_addr = 0;
-        return NULL;
-    }
-
-    rel_addr = (uintptr_t)addr - codemap->key;
-    data = (codemap_data_t *)codemap->data;
-    if (rel_addr >= data->machine_code_size) {
-        if (start_addr)
-            *start_addr = 0;
-        return NULL;
-    }
-
-    if (start_addr)
-        *start_addr = (long)codemap->key;
-    return (void *)codemap;
-}
-
-RPY_EXTERN
-long pypy_yield_codemap_at_addr(void *codemap_raw, long addr,
-                                long *current_pos_addr)
-{
-    // will return consecutive unique_ids from codemap, starting from position
-    // `pos` until addr
-    skipnode_t *codemap = (skipnode_t *)codemap_raw;
-    long current_pos = *current_pos_addr;
-    long rel_addr = addr - codemap->key;
-    long next_start, next_stop;
-    codemap_data_t *data = (codemap_data_t *)codemap->data;
-
-    while (1) {
-        if (current_pos >= data->bytecode_info_size)
-            return 0;
-        next_start = data->bytecode_info[current_pos + 1];
-        if (next_start > rel_addr)
-            return 0;
-        next_stop = data->bytecode_info[current_pos + 2];
-        if (next_stop > rel_addr) {
-            *current_pos_addr = current_pos + 4;
-            return data->bytecode_info[current_pos];
-        }
-        // we need to skip potentially more than one
-        current_pos = data->bytecode_info[current_pos + 3];
-    }
-}
-
-/************************************************************/
-/***  depthmap storage                                    ***/
-/************************************************************/
-
-typedef struct {
-    unsigned int block_size;
-    unsigned int stack_depth;
-} depthmap_data_t;
-
-static skipnode_t jit_depthmap_head;
-
-/*** interface used from codemap.py ***/
-
-RPY_EXTERN
-long pypy_jit_depthmap_add(uintptr_t addr, unsigned int size,
-                           unsigned int stackdepth)
-{
-    skipnode_t *new = skiplist_malloc(sizeof(depthmap_data_t));
-    depthmap_data_t *data;
-    if (new == NULL)
-        return -1;   /* too bad */
-
-    new->key = addr;
-    data = (depthmap_data_t *)new->data;
-    data->block_size = size;
-    data->stack_depth = stackdepth;
-
-    pypy_codemap_invalid_set(1);
-    skiplist_insert(&jit_depthmap_head, new);
-    pypy_codemap_invalid_set(0);
-    return 0;
-}
-
-RPY_EXTERN
-void pypy_jit_depthmap_clear(uintptr_t addr, unsigned int size)
-{
-    uintptr_t search_key = addr + size - 1;
-    if (size == 0)
-        return;
-
-    pypy_codemap_invalid_set(1);
-    while (1) {
-        /* search for all nodes belonging to the range, and remove them */
-        skipnode_t *node = skiplist_search(&jit_depthmap_head, search_key);
-        if (node->key < addr)
-            break;   /* exhausted */
-        skiplist_remove(&jit_depthmap_head, node->key);
-        free(node);
-    }
-    pypy_codemap_invalid_set(0);
-}
-
-/*** interface used from pypy/module/_vmprof ***/
-
-RPY_EXTERN
-long pypy_jit_stack_depth_at_loc(long loc)
-{
-    skipnode_t *depthmap = skiplist_search(&jit_depthmap_head, (uintptr_t)loc);
-    depthmap_data_t *data;
-    uintptr_t rel_addr;
-
-    if (depthmap == &jit_depthmap_head)
-        return -1;
-
-    rel_addr = (uintptr_t)loc - depthmap->key;
-    data = (codemap_data_t *)depthmap->data;
-    if (rel_addr >= data->block_size)
-        return -1;
-
-    return data->stack_depth;
-}
-
-/************************************************************/
diff --git a/rpython/jit/backend/llsupport/src/skiplist.c b/rpython/jit/backend/llsupport/src/skiplist.c
deleted file mode 100644
--- a/rpython/jit/backend/llsupport/src/skiplist.c
+++ /dev/null
@@ -1,103 +0,0 @@
-#include 
-#include 
-
-#define HAS_SKIPLIST
-#define SKIPLIST_HEIGHT   8
-
-typedef struct skipnode_s {
-    uintptr_t key;
-    char *data;
-    struct skipnode_s *next[SKIPLIST_HEIGHT];   /* may be smaller */
-} skipnode_t;
-
-static skipnode_t *skiplist_malloc(uintptr_t datasize)
-{
-    char *result;
-    uintptr_t basesize;
-    uintptr_t length = 1;
-    while (length < SKIPLIST_HEIGHT && (rand() & 3) == 0)
-        length++;
-    basesize = sizeof(skipnode_t) -
-               (SKIPLIST_HEIGHT - length) * sizeof(skipnode_t *);
-    result = malloc(basesize + datasize);
-    if (result != NULL) {
-        ((skipnode_t *)result)->data = result + basesize;
-    }
-    return (skipnode_t *)result;
-}
-
-static skipnode_t *skiplist_search(skipnode_t *head, uintptr_t searchkey)
-{
-    /* Returns the skipnode with key closest (but <=) searchkey.
-       Note that if there is no item with key <= searchkey in the list,
-       this will return the head node. */
-    uintptr_t level = SKIPLIST_HEIGHT - 1;
-    while (1) {
-        skipnode_t *next = head->next[level];
-        if (next != NULL && next->key <= searchkey) {
-            head = next;
-        }
-        else {
-            if (level == 0)
-                break;
-            level -= 1;
-        }
-    }
-    return head;
-}
-
-static void skiplist_insert(skipnode_t *head, skipnode_t *new)
-{
-    uintptr_t size0 = sizeof(skipnode_t) -
-                      SKIPLIST_HEIGHT * sizeof(skipnode_t *);
-    uintptr_t height_of_new = (new->data - ((char *)new + size0)) /
-                              sizeof(skipnode_t *);
-
-    uintptr_t level = SKIPLIST_HEIGHT - 1;
-    uintptr_t searchkey = new->key;
-    while (1) {
-        skipnode_t *next = head->next[level];
-        if (next != NULL && next->key <= searchkey) {
-            head = next;
-        }
-        else {
-            if (level < height_of_new) {
-                new->next[level] = next;
-                head->next[level] = new;
-                if (level == 0)
-                    break;
-            }
-            level -= 1;
-        }
-    }
-}
-
-static skipnode_t *skiplist_remove(skipnode_t *head, uintptr_t exact_key)
-{
-    uintptr_t level = SKIPLIST_HEIGHT - 1;
-    while (1) {
-        skipnode_t *next = head->next[level];
-        if (next != NULL && next->key <= exact_key) {
-            if (next->key == exact_key) {
-                head->next[level] = next->next[level];
-                if (level == 0)
-                    return next;    /* successfully removed */
-                level -= 1;
-            }
-            else
-                head = next;
-        }
-        else {
-            if (level == 0)
-                return NULL;    /* 'exact_key' not found! */
-            level -= 1;
-        }
-    }
-}
-
-static uintptr_t skiplist_firstkey(skipnode_t *head)
-{
-    if (head->next[0] == NULL)
-        return 0;
-    return head->next[0]->key;
-}
diff --git a/rpython/jit/backend/llsupport/test/test_asmmemmgr.py b/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
--- a/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
@@ -2,7 +2,7 @@
 from rpython.jit.backend.llsupport.asmmemmgr import AsmMemoryManager
 from rpython.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
 from rpython.jit.backend.llsupport.asmmemmgr import BlockBuilderMixin
-from rpython.jit.backend.llsupport.codemap import CodemapStorage
+from rpython.jit.backend.llsupport import asmmemmgr
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib import debug
 
@@ -96,21 +96,20 @@
 class TestAsmMemoryManager:
 
     def setup_method(self, _):
-        self.asmmemmgr = AsmMemoryManager(min_fragment=8,
+        self.memmgr = AsmMemoryManager(min_fragment=8,
                                        num_indices=10,
                                        large_alloc_size=8192)
-        self.codemap = CodemapStorage()
 
     def teardown_method(self, _):
-        self.asmmemmgr._delete()
+        self.memmgr._delete()
 
     def test_malloc_simple(self):
         for i in range(100):
-            while self.asmmemmgr.total_memory_allocated < 16384:
+            while self.memmgr.total_memory_allocated < 16384:
                 reqsize = random.randrange(1, 200)
-                (start, stop) = self.asmmemmgr.malloc(reqsize, reqsize)
+                (start, stop) = self.memmgr.malloc(reqsize, reqsize)
                 assert reqsize <= stop - start < reqsize + 8
-                assert self.asmmemmgr.total_memory_allocated in [8192, 16384]
+                assert self.memmgr.total_memory_allocated in [8192, 16384]
             self.teardown_method(None)
             self.setup_method(None)
 
@@ -124,7 +123,7 @@
             if got and (random.random() < 0.4 or len(got) == 1000):
                 # free
                 start, stop = got.pop(random.randrange(0, len(got)))
-                self.asmmemmgr.free(start, stop)
+                self.memmgr.free(start, stop)
                 real_use -= (stop - start)
                 assert real_use >= 0
             #
@@ -135,18 +134,18 @@
                     reqmaxsize = reqsize
                 else:
                     reqmaxsize = reqsize + random.randrange(0, 200)
-                (start, stop) = self.asmmemmgr.malloc(reqsize, reqmaxsize)
+                (start, stop) = self.memmgr.malloc(reqsize, reqmaxsize)
                 assert reqsize <= stop - start < reqmaxsize + 8
                 for otherstart, otherstop in got:           # no overlap
                     assert otherstop <= start or stop <= otherstart
                 got.append((start, stop))
                 real_use += (stop - start)
-                if self.asmmemmgr.total_memory_allocated == prev_total:
+                if self.memmgr.total_memory_allocated == prev_total:
                     iterations_without_allocating_more += 1
                     if iterations_without_allocating_more == 40000:
                         break    # ok
                 else:
-                    new_total = self.asmmemmgr.total_memory_allocated
+                    new_total = self.memmgr.total_memory_allocated
                     iterations_without_allocating_more = 0
                     print real_use, new_total
                     # We seem to never see a printed value greater
@@ -173,7 +172,7 @@
         #
         gcrootmap = FakeGcRootMap()
         allblocks = []
-        rawstart = mc.materialize(self, allblocks, gcrootmap)
+        rawstart = mc.materialize(self.memmgr, allblocks, gcrootmap)
         p = rffi.cast(rffi.CArrayPtr(lltype.Char), rawstart)
         assert p[0] == 'X'
         assert p[1] == 'x'
@@ -269,3 +268,16 @@
     md.done()
     assert allblocks == [(1597, 1697), (1797, 1835)]
     assert ops == [('free', 1835, 1897)]
+
+def test_find_jit_frame_depth():
+    mgr = AsmMemoryManager()
+    mgr.register_frame_depth_map(11, [0, 5, 10], [1, 2, 3])
+    mgr.register_frame_depth_map(30, [0, 5, 10], [4, 5, 6])
+    mgr.register_frame_depth_map(0, [0, 5, 10], [7, 8, 9])
+    asmmemmgr._memmngr = mgr
+    assert asmmemmgr.stack_depth_at_loc(13) == 1
+    assert asmmemmgr.stack_depth_at_loc(-3) == -1
+    assert asmmemmgr.stack_depth_at_loc(41) == -1
+    assert asmmemmgr.stack_depth_at_loc(5) == 8
+    assert asmmemmgr.stack_depth_at_loc(17) == 2
+    assert asmmemmgr.stack_depth_at_loc(38) == 5
diff --git a/rpython/jit/backend/llsupport/test/test_codemap.py b/rpython/jit/backend/llsupport/test/test_codemap.py
deleted file mode 100644
--- a/rpython/jit/backend/llsupport/test/test_codemap.py
+++ /dev/null
@@ -1,93 +0,0 @@
-
-from rpython.rtyper.lltypesystem import rffi, lltype
-from rpython.jit.backend.llsupport.codemap import stack_depth_at_loc
-from rpython.jit.backend.llsupport.codemap import CodemapStorage, \
-     CodemapBuilder, unpack_traceback, find_codemap_at_addr
-
-NULL = lltype.nullptr(rffi.CArray(lltype.Signed))
-     
-def test_register_codemap():
-    codemap = CodemapStorage()
-    codemap.setup()
-    codemap.register_codemap((100, 20, [13, 14, 15]))
-    codemap.register_codemap((300, 30, [16, 17, 18]))
-    codemap.register_codemap((200, 100, [19, 20, 21, 22, 23]))
-    #
-    raw100 = find_codemap_at_addr(100, NULL)
-    assert find_codemap_at_addr(119, NULL) == raw100
-    assert not find_codemap_at_addr(120, NULL)
-    #
-    raw200 = find_codemap_at_addr(200, NULL)
-    assert raw200 != raw100
-    assert find_codemap_at_addr(299, NULL) == raw200
-    #
-    raw300 = find_codemap_at_addr(329, NULL)
-    assert raw300 != raw100 and raw300 != raw200
-    assert find_codemap_at_addr(300, NULL) == raw300
-    #
-    codemap.free()
-
-def test_find_jit_frame_depth():
-    codemap = CodemapStorage()
-    codemap.setup()
-    codemap.register_frame_depth_map(11, 26, [0, 5, 10], [1, 2, 3])
-    codemap.register_frame_depth_map(30, 41, [0, 5, 10], [4, 5, 6])
-    codemap.register_frame_depth_map(0, 11, [0, 5, 10], [7, 8, 9])
-    assert stack_depth_at_loc(13) == 1
-    assert stack_depth_at_loc(-3) == -1
-    assert stack_depth_at_loc(40) == 6
-    assert stack_depth_at_loc(41) == -1
-    assert stack_depth_at_loc(5) == 8
-    assert stack_depth_at_loc(17) == 2
-    assert stack_depth_at_loc(38) == 5
-    assert stack_depth_at_loc(25) == 3
-    assert stack_depth_at_loc(26) == -1
-    assert stack_depth_at_loc(11) == 1
-    assert stack_depth_at_loc(10) == 9
-    codemap.free_asm_block(11, 26)
-    assert stack_depth_at_loc(11) == -1
-    assert stack_depth_at_loc(13) == -1
-    assert stack_depth_at_loc(-3) == -1
-    assert stack_depth_at_loc(40) == 6
-    assert stack_depth_at_loc(41) == -1
-    assert stack_depth_at_loc(5) == 8
-    assert stack_depth_at_loc(38) == 5
-    assert stack_depth_at_loc(10) == 9
-    codemap.free()
-
-def test_codemaps():
-    builder = CodemapBuilder()
-    builder.debug_merge_point(0, 102, 0)
-    builder.debug_merge_point(0, 102, 13)
-    builder.debug_merge_point(1, 104, 15)
-    builder.debug_merge_point(1, 104, 16)
-    builder.debug_merge_point(2, 106, 20)
-    builder.debug_merge_point(2, 106, 25)
-    builder.debug_merge_point(1, 104, 30)
-    builder.debug_merge_point(0, 102, 35)
-    codemap = CodemapStorage()
-    codemap.setup()
-    codemap.register_codemap(builder.get_final_bytecode(100, 40))
-    builder = CodemapBuilder()
-    builder.debug_merge_point(0, 202, 0)
-    builder.debug_merge_point(0, 202, 10)
-    builder.debug_merge_point(1, 204, 20)
-    builder.debug_merge_point(1, 204, 30)
-    builder.debug_merge_point(2, 206, 40)
-    builder.debug_merge_point(2, 206, 50)
-    builder.debug_merge_point(1, 204, 60)
-    builder.debug_merge_point(0, 202, 70)
-    codemap.register_codemap(builder.get_final_bytecode(200, 100))
-    assert unpack_traceback(110) == [102]
-    assert unpack_traceback(117) == [102, 104]
-    assert unpack_traceback(121) == [102, 104, 106]
-    assert unpack_traceback(131) == [102, 104]
-    assert unpack_traceback(137) == [102]
-    assert unpack_traceback(205) == [202]
-    assert unpack_traceback(225) == [202, 204]
-    assert unpack_traceback(245) == [202, 204, 206]
-    assert unpack_traceback(265) == [202, 204]
-    assert unpack_traceback(275) == [202]

From noreply at buildbot.pypy.org  Fri Apr 17 10:01:13 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 17 Apr 2015 10:01:13 +0200 (CEST)
Subject: [pypy-commit] pypy default: completely backout vmprof merge
Message-ID: <20150417080113.1C3961C1331@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r76812:a69d4a5a9638
Date: 2015-04-17 10:00 +0200
http://bitbucket.org/pypy/pypy/changeset/a69d4a5a9638/

Log:	completely backout vmprof merge

diff too long, truncating to 2000 out of 2674 lines

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -420,10 +420,3 @@
 the terms of the GPL license version 2 or any later version.  Thus the
 gdbm module, provided in the file lib_pypy/gdbm.py, is redistributed
 under the terms of the GPL license as well.
-
-License for 'pypy/module/_vmprof/src'
---------------------------------------
-
-The code is based on gperftools. You may see a copy of the License for it at
-
-    https://code.google.com/p/gperftools/source/browse/COPYING
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -38,9 +38,6 @@
     "_csv", "cppyy", "_pypyjson"
 ])
 
-if sys.platform.startswith('linux') and sys.maxint > 2147483647:
-    working_modules.add('_vmprof')
-
 translation_modules = default_modules.copy()
 translation_modules.update([
     "fcntl", "time", "select", "signal", "_rawffi", "zlib", "struct", "_md5",
@@ -102,7 +99,6 @@
     "_hashlib"  : ["pypy.module._ssl.interp_ssl"],
     "_minimal_curses": ["pypy.module._minimal_curses.fficurses"],
     "_continuation": ["rpython.rlib.rstacklet"],
-    "_vmprof" : ["pypy.module._vmprof.interp_vmprof"],
     }
 
 def get_module_validator(modname):
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -9,7 +9,6 @@
 from rpython.rlib.signature import signature
 from rpython.rlib.rarithmetic import r_uint, SHRT_MIN, SHRT_MAX, \
     INT_MIN, INT_MAX, UINT_MAX, USHRT_MAX
-from rpython.rlib.rweaklist import RWeakListMixin
 
 from pypy.interpreter.executioncontext import (ExecutionContext, ActionFlag,
     UserDelAction)
@@ -367,10 +366,6 @@
 
 # ____________________________________________________________
 
-class CodeObjWeakList(RWeakListMixin):
-    def __init__(self):
-        self.initialize()
-
 class ObjSpace(object):
     """Base class for the interpreter-level implementations of object spaces.
     http://pypy.readthedocs.org/en/latest/objspace.html"""
@@ -394,7 +389,6 @@
         self.check_signal_action = None   # changed by the signal module
         self.user_del_action = UserDelAction(self)
         self._code_of_sys_exc_info = None
-        self.all_code_objs = CodeObjWeakList()
 
         # can be overridden to a subclass
         self.initialize()
@@ -672,16 +666,6 @@
             assert ec is not None
             return ec
 
-    def register_code_object(self, pycode):
-        callback = self.getexecutioncontext().register_code_callback
-        if callback is not None:
-            callback(self, pycode)
-        self.all_code_objs.add_handle(pycode)
-
-    def set_code_callback(self, callback):
-        ec = self.getexecutioncontext()
-        ec.register_code_callback = callback
-        
     def _freeze_(self):
         return True
 
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -33,11 +33,6 @@
         self.profilefunc = None
         self.w_profilefuncarg = None
         self.thread_disappeared = False   # might be set to True after os.fork()
-        self.register_code_callback = None
-        if sys.maxint == 2147483647:
-            self._code_unique_id = 0 # XXX this is wrong, it won't work on 32bit
-        else:
-            self._code_unique_id = 0x7000000000000000
 
     @staticmethod
     def _mark_thread_disappeared(space):
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -14,10 +14,9 @@
     CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS, CO_NESTED,
     CO_GENERATOR, CO_KILL_DOCSTRING, CO_YIELD_INSIDE_TRY)
 from pypy.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT
-from rpython.rlib.rarithmetic import intmask, r_longlong
+from rpython.rlib.rarithmetic import intmask
 from rpython.rlib.objectmodel import compute_hash
 from rpython.rlib import jit
-from rpython.rlib.debug import debug_start, debug_stop, debug_print
 
 
 class BytecodeCorruption(Exception):
@@ -55,9 +54,8 @@
     "CPython-style code objects."
     _immutable_ = True
     _immutable_fields_ = ["co_consts_w[*]", "co_names_w[*]", "co_varnames[*]",
-                          "co_freevars[*]", "co_cellvars[*]",
-                          "_args_as_cellvars[*]"]
-    
+                          "co_freevars[*]", "co_cellvars[*]", "_args_as_cellvars[*]"]
+
     def __init__(self, space,  argcount, nlocals, stacksize, flags,
                      code, consts, names, varnames, filename,
                      name, firstlineno, lnotab, freevars, cellvars,
@@ -85,7 +83,6 @@
         self.magic = magic
         self._signature = cpython_code_signature(self)
         self._initialize()
-        space.register_code_object(self)
 
     def _initialize(self):
         if self.co_cellvars:
@@ -127,15 +124,6 @@
             from pypy.objspace.std.mapdict import init_mapdict_cache
             init_mapdict_cache(self)
 
-        ec = self.space.getexecutioncontext()
-        self._unique_id = ec._code_unique_id
-        ec._code_unique_id += 2 # so we have one bit that we can mark stuff
-        # with
-
-    def _get_full_name(self):
-        return "py:%s:%d:%s" % (self.co_name, self.co_firstlineno,
-                                self.co_filename)
-
     def _cleanup_(self):
         if (self.magic == cpython_magic and
             '__pypy__' not in sys.builtin_module_names):
diff --git a/pypy/module/_vmprof/__init__.py b/pypy/module/_vmprof/__init__.py
deleted file mode 100644
--- a/pypy/module/_vmprof/__init__.py
+++ /dev/null
@@ -1,18 +0,0 @@
-from pypy.interpreter.mixedmodule import MixedModule
-
-class Module(MixedModule):
-    """
-    Write me :)
-    """
-    appleveldefs = {
-    }
-
-    interpleveldefs = {
-        'enable': 'interp_vmprof.enable',
-        'disable': 'interp_vmprof.disable',
-    }
-
-    def setup_after_space_initialization(self):
-        # force the __extend__ hacks to occur early
-        from pypy.module._vmprof.interp_vmprof import VMProf
-        self.vmprof = VMProf()
diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
deleted file mode 100644
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ /dev/null
@@ -1,223 +0,0 @@
-import py, os, sys
-from rpython.rtyper.lltypesystem import lltype, rffi, llmemory
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.rtyper.annlowlevel import cast_instance_to_gcref, cast_base_ptr_to_instance
-from rpython.rlib.objectmodel import we_are_translated
-from rpython.rlib import jit, rposix, entrypoint
-from rpython.rtyper.tool import rffi_platform as platform
-from rpython.rlib.rstring import StringBuilder
-from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.error import oefmt, wrap_oserror, OperationError
-from pypy.interpreter.gateway import unwrap_spec
-from pypy.interpreter.pyframe import PyFrame
-
-ROOT = py.path.local(__file__).join('..')
-SRC = ROOT.join('src')
-
-# by default, we statically link vmprof.c into pypy; however, if you set
-# DYNAMIC_VMPROF to True, it will be dynamically linked to the libvmprof.so
-# which is expected to be inside pypy/module/_vmprof/src: this is very useful
-# during development. Note that you have to manually build libvmprof by
-# running make inside the src dir
-DYNAMIC_VMPROF = False
-
-eci_kwds = dict(
-    include_dirs = [SRC],
-    includes = ['vmprof.h', 'trampoline.h'],
-    separate_module_files = [SRC.join('trampoline.asmgcc.s')],
-    libraries = ['unwind'],
-    
-    post_include_bits=["""
-        void* pypy_vmprof_get_virtual_ip(void*);
-        void pypy_vmprof_init(void);
-    """],
-    
-    separate_module_sources=["""
-        void pypy_vmprof_init(void) {
-            vmprof_set_mainloop(pypy_execute_frame_trampoline, 0,
-                                pypy_vmprof_get_virtual_ip);
-        }
-    """],
-    )
-
-
-if DYNAMIC_VMPROF:
-    eci_kwds['libraries'] += ['vmprof']
-    eci_kwds['link_extra'] = ['-Wl,-rpath,%s' % SRC, '-L%s' % SRC]
-else:
-    eci_kwds['separate_module_files'] += [SRC.join('vmprof.c')]
-
-eci = ExternalCompilationInfo(**eci_kwds)
-
-check_eci = eci.merge(ExternalCompilationInfo(separate_module_files=[
-    SRC.join('fake_pypy_api.c')]))
-
-platform.verify_eci(check_eci)
-
-pypy_execute_frame_trampoline = rffi.llexternal(
-    "pypy_execute_frame_trampoline",
-    [llmemory.GCREF, llmemory.GCREF, llmemory.GCREF],
-    llmemory.GCREF,
-    compilation_info=eci,
-    _nowrapper=True, sandboxsafe=True,
-    random_effects_on_gcobjs=True)
-
-pypy_vmprof_init = rffi.llexternal("pypy_vmprof_init", [], lltype.Void,
-                                   compilation_info=eci)
-vmprof_enable = rffi.llexternal("vmprof_enable",
-                                [rffi.INT, rffi.LONG, rffi.INT,
-                                 rffi.CCHARP, rffi.INT],
-                                rffi.INT, compilation_info=eci,
-                                save_err=rffi.RFFI_SAVE_ERRNO)
-vmprof_disable = rffi.llexternal("vmprof_disable", [], rffi.INT,
-                                 compilation_info=eci,
-                                save_err=rffi.RFFI_SAVE_ERRNO)
-
-vmprof_register_virtual_function = rffi.llexternal(
-    "vmprof_register_virtual_function",
-    [rffi.CCHARP, rffi.VOIDP, rffi.VOIDP], lltype.Void,
-    compilation_info=eci, _nowrapper=True)
-
-original_execute_frame = PyFrame.execute_frame.im_func
-original_execute_frame.c_name = 'pypy_pyframe_execute_frame'
-original_execute_frame._dont_inline_ = True
-
-class __extend__(PyFrame):
-    def execute_frame(frame, w_inputvalue=None, operr=None):
-        # go through the asm trampoline ONLY if we are translated but not being JITted.
-        #
-        # If we are not translated, we obviously don't want to go through the
-        # trampoline because there is no C function it can call.
-        #
-        # If we are being JITted, we want to skip the trampoline, else the JIT
-        # cannot see throug it
-        if we_are_translated() and not jit.we_are_jitted():
-            # if we are translated, call the trampoline
-            gc_frame = cast_instance_to_gcref(frame)
-            gc_inputvalue = cast_instance_to_gcref(w_inputvalue)
-            gc_operr = cast_instance_to_gcref(operr)
-            gc_result = pypy_execute_frame_trampoline(gc_frame, gc_inputvalue, gc_operr)
-            return cast_base_ptr_to_instance(W_Root, gc_result)
-        else:
-            return original_execute_frame(frame, w_inputvalue, operr)
-
-
- at entrypoint.entrypoint_lowlevel('main', [llmemory.GCREF],
-                                'pypy_vmprof_get_virtual_ip', True)
-def get_virtual_ip(gc_frame):
-    frame = cast_base_ptr_to_instance(PyFrame, gc_frame)
-    if jit._get_virtualizable_token(frame):
-        return rffi.cast(rffi.VOIDP, 0)
-    virtual_ip = do_get_virtual_ip(frame)
-    return rffi.cast(rffi.VOIDP, virtual_ip)
-
-def do_get_virtual_ip(frame):
-    return frame.pycode._unique_id
-
-def write_long_to_string_builder(l, b):
-    if sys.maxint == 2147483647:
-        b.append(chr(l & 0xff))
-        b.append(chr((l >> 8) & 0xff))
-        b.append(chr((l >> 16) & 0xff))
-        b.append(chr((l >> 24) & 0xff))
-    else:
-        b.append(chr(l & 0xff))
-        b.append(chr((l >> 8) & 0xff))
-        b.append(chr((l >> 16) & 0xff))
-        b.append(chr((l >> 24) & 0xff))
-        b.append(chr((l >> 32) & 0xff))
-        b.append(chr((l >> 40) & 0xff))
-        b.append(chr((l >> 48) & 0xff))
-        b.append(chr((l >> 56) & 0xff))
-
-class VMProf(object):
-    def __init__(self):
-        self.is_enabled = False
-        self.ever_enabled = False
-        self.mapping_so_far = [] # stored mapping in between runs
-        self.fileno = -1
-
-    def enable(self, space, fileno, period):
-        if self.is_enabled:
-            raise oefmt(space.w_ValueError, "_vmprof already enabled")
-        self.fileno = fileno
-        self.is_enabled = True
-        self.write_header(fileno, period)
-        if not self.ever_enabled:
-            if we_are_translated():
-                pypy_vmprof_init()
-            self.ever_enabled = True
-        for weakcode in space.all_code_objs.get_all_handles():
-            code = weakcode()
-            if code:
-                self.register_code(space, code)
-        space.set_code_callback(vmprof_register_code)
-        if we_are_translated():
-            # does not work untranslated
-            res = vmprof_enable(fileno, period, 0,
-                                lltype.nullptr(rffi.CCHARP.TO), 0)
-        else:
-            res = 0
-        if res == -1:
-            raise wrap_oserror(space, OSError(rposix.get_saved_errno(),
-                                              "_vmprof.enable"))
-
-    def write_header(self, fileno, period):
-        if period == -1:
-            period_usec = 1000000 / 100 #  100hz
-        else:
-            period_usec = period
-        b = StringBuilder()
-        write_long_to_string_builder(0, b)
-        write_long_to_string_builder(3, b)
-        write_long_to_string_builder(0, b)
-        write_long_to_string_builder(period_usec, b)
-        write_long_to_string_builder(0, b)
-        os.write(fileno, b.build())
-
-    def register_code(self, space, code):
-        if self.fileno == -1:
-            raise OperationError(space.w_RuntimeError,
-                                 space.wrap("vmprof not running"))
-        name = code._get_full_name()
-        b = StringBuilder()
-        b.append('\x02')
-        write_long_to_string_builder(code._unique_id, b)
-        write_long_to_string_builder(len(name), b)
-        b.append(name)
-        os.write(self.fileno, b.build())
-
-    def disable(self, space):
-        if not self.is_enabled:
-            raise oefmt(space.w_ValueError, "_vmprof not enabled")
-        self.is_enabled = False
-        self.fileno = -1
-        if we_are_translated():
-           # does not work untranslated
-            res = vmprof_disable()
-        else:
-            res = 0
-        space.set_code_callback(None)
-        if res == -1:
-            raise wrap_oserror(space, OSError(rposix.get_saved_errno(),
-                                              "_vmprof.disable"))
-
-def vmprof_register_code(space, code):
-    from pypy.module._vmprof import Module
-    mod_vmprof = space.getbuiltinmodule('_vmprof')
-    assert isinstance(mod_vmprof, Module)
-    mod_vmprof.vmprof.register_code(space, code)
-        
- at unwrap_spec(fileno=int, period=int)
-def enable(space, fileno, period=-1):
-    from pypy.module._vmprof import Module
-    mod_vmprof = space.getbuiltinmodule('_vmprof')
-    assert isinstance(mod_vmprof, Module)
-    mod_vmprof.vmprof.enable(space, fileno, period)
-
-def disable(space):
-    from pypy.module._vmprof import Module
-    mod_vmprof = space.getbuiltinmodule('_vmprof')
-    assert isinstance(mod_vmprof, Module)
-    mod_vmprof.vmprof.disable(space)
-
diff --git a/pypy/module/_vmprof/src/config.h b/pypy/module/_vmprof/src/config.h
deleted file mode 100644
--- a/pypy/module/_vmprof/src/config.h
+++ /dev/null
@@ -1,2 +0,0 @@
-#define HAVE_SYS_UCONTEXT_H
-#define PC_FROM_UCONTEXT uc_mcontext.gregs[REG_RIP]
diff --git a/pypy/module/_vmprof/src/fake_pypy_api.c b/pypy/module/_vmprof/src/fake_pypy_api.c
deleted file mode 100644
--- a/pypy/module/_vmprof/src/fake_pypy_api.c
+++ /dev/null
@@ -1,29 +0,0 @@
-
-long pypy_jit_start_addr(void)
-{
-	return 3;
-}
-
-long pypy_jit_end_addr(void)
-{
-	return 3;
-}
-
-long pypy_jit_stack_depth_at_loc(long x)
-{
-	return 0;
-}
-
-long pypy_find_codemap_at_addr(long x)
-{
-	return 0;
-}
-
-long pypy_yield_codemap_at_addr(long x, long y, long *a)
-{
-	return 0;
-}
-
-void pypy_pyframe_execute_frame(void)
-{
-}
diff --git a/pypy/module/_vmprof/src/get_custom_offset.c b/pypy/module/_vmprof/src/get_custom_offset.c
deleted file mode 100644
--- a/pypy/module/_vmprof/src/get_custom_offset.c
+++ /dev/null
@@ -1,46 +0,0 @@
-
-long pypy_jit_start_addr();
-long pypy_jit_end_addr();
-long pypy_jit_stack_depth_at_loc(long);
-long pypy_find_codemap_at_addr(long);
-long pypy_yield_codemap_at_addr(long, long, long*);
-
-void vmprof_set_tramp_range(void* start, void* end)
-{
-}
-
-static ptrdiff_t vmprof_unw_get_custom_offset(void* ip, unw_cursor_t *cp) {
-	intptr_t ip_l = (intptr_t)ip;
-
-	if (ip_l < pypy_jit_start_addr() || ip_l > pypy_jit_end_addr()) {
-		return -1;
-	}
-	return (void*)pypy_jit_stack_depth_at_loc(ip_l);
-}
-
-static long vmprof_write_header_for_jit_addr(void **result, long n,
-											 void *ip, int max_depth)
-{
-	long codemap_pos;
-	long current_pos = 0;
-	intptr_t id;
-	intptr_t addr = (intptr_t)ip;
-
-	if (addr < pypy_jit_start_addr() || addr > pypy_jit_end_addr()) {
-		return n;
-	}
-	codemap_pos = pypy_find_codemap_at_addr(addr);
-	if (codemap_pos == -1) {
-		return n;
-	}
-	while (1) {
-		id = pypy_yield_codemap_at_addr(codemap_pos, addr, ¤t_pos);
-		if (id == 0) {
-			return n;
-		}
-		result[n++] = id;
-		if (n >= max_depth) {
-			return n;
-		}
-	}
-}
diff --git a/pypy/module/_vmprof/src/getpc.h b/pypy/module/_vmprof/src/getpc.h
deleted file mode 100644
--- a/pypy/module/_vmprof/src/getpc.h
+++ /dev/null
@@ -1,187 +0,0 @@
-// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*-
-// Copyright (c) 2005, Google Inc.
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-//     * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-//     * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-//     * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-// ---
-// Author: Craig Silverstein
-//
-// This is an internal header file used by profiler.cc.  It defines
-// the single (inline) function GetPC.  GetPC is used in a signal
-// handler to figure out the instruction that was being executed when
-// the signal-handler was triggered.
-//
-// To get this, we use the ucontext_t argument to the signal-handler
-// callback, which holds the full context of what was going on when
-// the signal triggered.  How to get from a ucontext_t to a Program
-// Counter is OS-dependent.
-
-#ifndef BASE_GETPC_H_
-#define BASE_GETPC_H_
-
-#include "config.h"
-
-// On many linux systems, we may need _GNU_SOURCE to get access to
-// the defined constants that define the register we want to see (eg
-// REG_EIP).  Note this #define must come first!
-#define _GNU_SOURCE 1
-// If #define _GNU_SOURCE causes problems, this might work instead.
-// It will cause problems for FreeBSD though!, because it turns off
-// the needed __BSD_VISIBLE.
-//#define _XOPEN_SOURCE 500
-
-#include          // for memcmp
-#if defined(HAVE_SYS_UCONTEXT_H)
-#include 
-#elif defined(HAVE_UCONTEXT_H)
-#include        // for ucontext_t (and also mcontext_t)
-#elif defined(HAVE_CYGWIN_SIGNAL_H)
-#include 
-typedef ucontext ucontext_t;
-#endif
-
-
-// Take the example where function Foo() calls function Bar().  For
-// many architectures, Bar() is responsible for setting up and tearing
-// down its own stack frame.  In that case, it's possible for the
-// interrupt to happen when execution is in Bar(), but the stack frame
-// is not properly set up (either before it's done being set up, or
-// after it's been torn down but before Bar() returns).  In those
-// cases, the stack trace cannot see the caller function anymore.
-//
-// GetPC can try to identify this situation, on architectures where it
-// might occur, and unwind the current function call in that case to
-// avoid false edges in the profile graph (that is, edges that appear
-// to show a call skipping over a function).  To do this, we hard-code
-// in the asm instructions we might see when setting up or tearing
-// down a stack frame.
-//
-// This is difficult to get right: the instructions depend on the
-// processor, the compiler ABI, and even the optimization level.  This
-// is a best effort patch -- if we fail to detect such a situation, or
-// mess up the PC, nothing happens; the returned PC is not used for
-// any further processing.
-struct CallUnrollInfo {
-  // Offset from (e)ip register where this instruction sequence
-  // should be matched. Interpreted as bytes. Offset 0 is the next
-  // instruction to execute. Be extra careful with negative offsets in
-  // architectures of variable instruction length (like x86) - it is
-  // not that easy as taking an offset to step one instruction back!
-  int pc_offset;
-  // The actual instruction bytes. Feel free to make it larger if you
-  // need a longer sequence.
-  unsigned char ins[16];
-  // How many bytes to match from ins array?
-  int ins_size;
-  // The offset from the stack pointer (e)sp where to look for the
-  // call return address. Interpreted as bytes.
-  int return_sp_offset;
-};
-
-
-// The dereferences needed to get the PC from a struct ucontext were
-// determined at configure time, and stored in the macro
-// PC_FROM_UCONTEXT in config.h.  The only thing we need to do here,
-// then, is to do the magic call-unrolling for systems that support it.
-
-// -- Special case 1: linux x86, for which we have CallUnrollInfo
-#if defined(__linux) && defined(__i386) && defined(__GNUC__)
-static const CallUnrollInfo callunrollinfo[] = {
-  // Entry to a function:  push %ebp;  mov  %esp,%ebp
-  // Top-of-stack contains the caller IP.
-  { 0,
-    {0x55, 0x89, 0xe5}, 3,
-    0
-  },
-  // Entry to a function, second instruction:  push %ebp;  mov  %esp,%ebp
-  // Top-of-stack contains the old frame, caller IP is +4.
-  { -1,
-    {0x55, 0x89, 0xe5}, 3,
-    4
-  },
-  // Return from a function: RET.
-  // Top-of-stack contains the caller IP.
-  { 0,
-    {0xc3}, 1,
-    0
-  }
-};
-
-inline void* GetPC(ucontext_t *signal_ucontext) {
-  // See comment above struct CallUnrollInfo.  Only try instruction
-  // flow matching if both eip and esp looks reasonable.
-  const int eip = signal_ucontext->uc_mcontext.gregs[REG_EIP];
-  const int esp = signal_ucontext->uc_mcontext.gregs[REG_ESP];
-  if ((eip & 0xffff0000) != 0 && (~eip & 0xffff0000) != 0 &&
-      (esp & 0xffff0000) != 0) {
-    char* eip_char = reinterpret_cast(eip);
-    for (int i = 0; i < sizeof(callunrollinfo)/sizeof(*callunrollinfo); ++i) {
-      if (!memcmp(eip_char + callunrollinfo[i].pc_offset,
-                  callunrollinfo[i].ins, callunrollinfo[i].ins_size)) {
-        // We have a match.
-        void **retaddr = (void**)(esp + callunrollinfo[i].return_sp_offset);
-        return *retaddr;
-      }
-    }
-  }
-  return (void*)eip;
-}
-
-// Special case #2: Windows, which has to do something totally different.
-#elif defined(_WIN32) || defined(__CYGWIN__) || defined(__CYGWIN32__) || defined(__MINGW32__)
-// If this is ever implemented, probably the way to do it is to have
-// profiler.cc use a high-precision timer via timeSetEvent:
-//    http://msdn2.microsoft.com/en-us/library/ms712713.aspx
-// We'd use it in mode TIME_CALLBACK_FUNCTION/TIME_PERIODIC.
-// The callback function would be something like prof_handler, but
-// alas the arguments are different: no ucontext_t!  I don't know
-// how we'd get the PC (using StackWalk64?)
-//    http://msdn2.microsoft.com/en-us/library/ms680650.aspx
-
-#include "base/logging.h"   // for RAW_LOG
-#ifndef HAVE_CYGWIN_SIGNAL_H
-typedef int ucontext_t;
-#endif
-
-inline void* GetPC(ucontext_t *signal_ucontext) {
-  RAW_LOG(ERROR, "GetPC is not yet implemented on Windows\n");
-  return NULL;
-}
-
-// Normal cases.  If this doesn't compile, it's probably because
-// PC_FROM_UCONTEXT is the empty string.  You need to figure out
-// the right value for your system, and add it to the list in
-// configure.ac (or set it manually in your config.h).
-#else
-inline void* GetPC(ucontext_t *signal_ucontext) {
-  return (void*)signal_ucontext->PC_FROM_UCONTEXT;   // defined in config.h
-}
-
-#endif
-
-#endif  // BASE_GETPC_H_
diff --git a/pypy/module/_vmprof/src/trampoline.asmgcc.s b/pypy/module/_vmprof/src/trampoline.asmgcc.s
deleted file mode 100644
--- a/pypy/module/_vmprof/src/trampoline.asmgcc.s
+++ /dev/null
@@ -1,17 +0,0 @@
-// NOTE: you need to use TABs, not spaces!
-        
-	.text
-	.p2align 4,,-1
-	.globl	pypy_execute_frame_trampoline
-	.type	pypy_execute_frame_trampoline, @function
-pypy_execute_frame_trampoline:
-	.cfi_startproc
-	pushq	%rdi
-	.cfi_def_cfa_offset 16
-	call pypy_pyframe_execute_frame at PLT
-	/* GCROOT 0(%rsp) */
-	popq	%rdi
-	.cfi_def_cfa_offset 8
-	ret
-	.cfi_endproc
-	.size	pypy_execute_frame_trampoline, .-pypy_execute_frame_trampoline
diff --git a/pypy/module/_vmprof/src/trampoline.h b/pypy/module/_vmprof/src/trampoline.h
deleted file mode 100644
--- a/pypy/module/_vmprof/src/trampoline.h
+++ /dev/null
@@ -1,1 +0,0 @@
-void* pypy_execute_frame_trampoline(void*, void*, void*);
diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c
deleted file mode 100644
--- a/pypy/module/_vmprof/src/vmprof.c
+++ /dev/null
@@ -1,340 +0,0 @@
-/* VMPROF
- *
- * statistical sampling profiler specifically designed to profile programs
- * which run on a Virtual Machine and/or bytecode interpreter, such as Python,
- * etc.
- *
- * The logic to dump the C stack traces is partly stolen from the code in gperftools.
- * The file "getpc.h" has been entirely copied from gperftools.
- *
- * Tested only on gcc, linux, x86_64.
- *
- * Copyright (C) 2014-2015
- *   Antonio Cuni - anto.cuni at gmail.com
- *   Maciej Fijalkowski - fijall at gmail.com
- *
- */
-
-
-#include "getpc.h"      // should be first to get the _GNU_SOURCE dfn
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-
-#define UNW_LOCAL_ONLY
-#include 
-
-#include "vmprof.h"
-
-#define _unused(x) ((void)x)
-
-#define MAX_FUNC_NAME 128
-#define MAX_STACK_DEPTH 64
-
-static FILE* profile_file = NULL;
-void* vmprof_mainloop_func;
-static ptrdiff_t mainloop_sp_offset;
-static vmprof_get_virtual_ip_t mainloop_get_virtual_ip;
-
-
-/* *************************************************************
- * functions to write a profile file compatible with gperftools
- * *************************************************************
- */
-
-#define MARKER_STACKTRACE '\x01'
-#define MARKER_VIRTUAL_IP '\x02'
-#define MARKER_TRAILER '\x03'
-
-static void prof_word(FILE* f, long x) {
-    fwrite(&x, sizeof(x), 1, f);
-}
-
-static void prof_header(FILE* f, long period_usec) {
-    prof_word(f, 0);
-    prof_word(f, 3);
-    prof_word(f, 0);
-    prof_word(f, period_usec);
-    prof_word(f, 0);
-}
-
-static void prof_write_stacktrace(FILE* f, void** stack, int depth, int count) {
-    int i;
-	char marker = MARKER_STACKTRACE;
-
-	fwrite(&marker, 1, 1, f);
-    prof_word(f, count);
-    prof_word(f, depth);
-    for(i=0; isp = bp;
-		bp -= sizeof(void*);
-        cp2->ip = ((void**)bp)[0];
-        // the ret is on the top of the stack minus WORD
-        return 1;
-    }
-}
-
-
-/* *************************************************************
- * functions to dump the stack trace
- * *************************************************************
- */
-
-// stolen from pprof:
-// Sometimes, we can try to get a stack trace from within a stack
-// trace, because libunwind can call mmap (maybe indirectly via an
-// internal mmap based memory allocator), and that mmap gets trapped
-// and causes a stack-trace request.  If were to try to honor that
-// recursive request, we'd end up with infinite recursion or deadlock.
-// Luckily, it's safe to ignore those subsequent traces.  In such
-// cases, we return 0 to indicate the situation.
-//static __thread int recursive;
-static int recursive; // XXX antocuni: removed __thread
-
-int get_stack_trace(void** result, int max_depth, ucontext_t *ucontext) {
-    void *ip;
-    int n = 0;
-    unw_cursor_t cursor;
-    unw_context_t uc = *ucontext;
-    if (recursive) {
-        return 0;
-    }
-    ++recursive;
-
-    int ret = unw_init_local(&cursor, &uc);
-    assert(ret >= 0);
-    _unused(ret);
-
-    while (n < max_depth) {
-        if (unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *) &ip) < 0) {
-            break;
-        }
-
-        unw_proc_info_t pip;
-        unw_get_proc_info(&cursor, &pip);
-
-        /* char funcname[4096]; */
-        /* unw_word_t offset; */
-        /* unw_get_proc_name(&cursor, funcname, 4096, &offset); */
-        /* printf("%s+%#lx <%p>\n", funcname, offset, ip); */
-
-        /* if n==0, it means that the signal handler interrupted us while we
-           were in the trampoline, so we are not executing (yet) the real main
-           loop function; just skip it */
-        if (vmprof_mainloop_func && 
-            (void*)pip.start_ip == (void*)vmprof_mainloop_func &&
-            n > 0) {
-          // found main loop stack frame
-          void* sp;
-          unw_get_reg(&cursor, UNW_REG_SP, (unw_word_t *) &sp);
-          void *arg_addr = (char*)sp + mainloop_sp_offset;
-          void **arg_ptr = (void**)arg_addr;
-          // fprintf(stderr, "stacktrace mainloop: rsp %p   &f2 %p   offset %ld\n", 
-          //         sp, arg_addr, mainloop_sp_offset);
-          ip = mainloop_get_virtual_ip(*arg_ptr);
-        }
-
-        result[n++] = ip;
-		n = vmprof_write_header_for_jit_addr(result, n, ip, max_depth);
-        if (vmprof_unw_step(&cursor) <= 0) {
-            break;
-        }
-    }
-    --recursive;
-    return n;
-}
-
-
-static int __attribute__((noinline)) frame_forcer(int rv) {
-    return rv;
-}
-
-static void sigprof_handler(int sig_nr, siginfo_t* info, void *ucontext) {
-    void* stack[MAX_STACK_DEPTH];
-    stack[0] = GetPC((ucontext_t*)ucontext);
-    int depth = frame_forcer(get_stack_trace(stack+1, MAX_STACK_DEPTH-1, ucontext));
-    depth++;  // To account for pc value in stack[0];
-    prof_write_stacktrace(profile_file, stack, depth, 1);
-}
-
-/* *************************************************************
- * functions to enable/disable the profiler
- * *************************************************************
- */
-
-static int open_profile(int fd, long period_usec, int write_header, char *s,
-						int slen) {
-	if ((fd = dup(fd)) == -1) {
-		return -1;
-	}
-    profile_file = fdopen(fd, "wb");
-	if (!profile_file) {
-		return -1;
-	}
-	if (write_header)
-		prof_header(profile_file, period_usec);
-	if (s)
-		fwrite(s, slen, 1, profile_file);
-	return 0;
-}
-
-static int close_profile(void) {
-	// XXX all of this can happily fail
-    FILE* src;
-    char buf[BUFSIZ];
-    size_t size;
-	int marker = MARKER_TRAILER;
-	fwrite(&marker, 1, 1, profile_file);
-
-    // copy /proc/PID/maps to the end of the profile file
-    sprintf(buf, "/proc/%d/maps", getpid());
-    src = fopen(buf, "r");    
-    while ((size = fread(buf, 1, BUFSIZ, src))) {
-        fwrite(buf, 1, size, profile_file);
-    }
-    fclose(src);
-    fclose(profile_file);
-	return 0;
-}
-
-
-static int install_sigprof_handler(void) {
-    struct sigaction sa;
-    memset(&sa, 0, sizeof(sa));
-    sa.sa_sigaction = sigprof_handler;
-    sa.sa_flags = SA_RESTART | SA_SIGINFO;
-    if (sigemptyset(&sa.sa_mask) == -1 ||
-		sigaction(SIGPROF, &sa, NULL) == -1) {
-		return -1;
-	}
-	return 0;
-}
-
-static int remove_sigprof_handler(void) {
-    //sighandler_t res = signal(SIGPROF, SIG_DFL);
-	//if (res == SIG_ERR) {
-	//	return -1;
-	//}
-	return 0;
-};
-
-static int install_sigprof_timer(long period_usec) {
-    static struct itimerval timer;
-    timer.it_interval.tv_sec = 0;
-    timer.it_interval.tv_usec = period_usec;
-    timer.it_value = timer.it_interval;
-    if (setitimer(ITIMER_PROF, &timer, NULL) != 0) {
-		return -1;
-    }
-	return 0;
-}
-
-static int remove_sigprof_timer(void) {
-    static struct itimerval timer;
-    timer.it_interval.tv_sec = 0;
-    timer.it_interval.tv_usec = 0;
-    timer.it_value = timer.it_interval;
-    if (setitimer(ITIMER_PROF, &timer, NULL) != 0) {
-		return -1;
-    }
-	return 0;
-}
-
-/* *************************************************************
- * public API
- * *************************************************************
- */
-
-void vmprof_set_mainloop(void* func, ptrdiff_t sp_offset, 
-                         vmprof_get_virtual_ip_t get_virtual_ip) {
-    mainloop_sp_offset = sp_offset;
-    mainloop_get_virtual_ip = get_virtual_ip;
-    vmprof_mainloop_func = func;
-}
-
-int vmprof_enable(int fd, long period_usec, int write_header, char *s,
-				  int slen)
-{
-    if (period_usec == -1)
-        period_usec = 1000000 / 100; /* 100hz */
-    if (open_profile(fd, period_usec, write_header, s, slen) == -1) {
-		return -1;
-	}
-    if (install_sigprof_handler() == -1) {
-		return -1;
-	}
-    if (install_sigprof_timer(period_usec) == -1) {
-		return -1;
-	}
-	return 0;
-}
-
-int vmprof_disable(void) {
-    if (remove_sigprof_timer() == -1) {
-		return -1;
-	}
-    if (remove_sigprof_handler() == -1) {
-		return -1;
-	}
-    if (close_profile() == -1) {
-		return -1;
-	}
-	return 0;
-}
-
-void vmprof_register_virtual_function(const char* name, void* start, void* end) {
-    // for now *end is simply ignored
-	char buf[1024];
-	int lgt = strlen(name) + 2 * sizeof(long) + 1;
-
-	if (lgt > 1024) {
-		lgt = 1024;
-	}
-	buf[0] = MARKER_VIRTUAL_IP;
-	((void **)(((void*)buf) + 1))[0] = start;
-	((long *)(((void*)buf) + 1 + sizeof(long)))[0] = lgt - 2 * sizeof(long) - 1;
-	strncpy(buf + 2 * sizeof(long) + 1, name, 1024 - 2 * sizeof(long) - 1);
-	fwrite(buf, lgt, 1, profile_file);
-}
diff --git a/pypy/module/_vmprof/src/vmprof.h b/pypy/module/_vmprof/src/vmprof.h
deleted file mode 100644
--- a/pypy/module/_vmprof/src/vmprof.h
+++ /dev/null
@@ -1,22 +0,0 @@
-#ifndef VMPROF_VMPROF_H_
-#define VMPROF_VMPROF_H_
-
-#include 
-
-typedef void* (*vmprof_get_virtual_ip_t)(void*);
-
-extern void* vmprof_mainloop_func;
-void vmprof_set_mainloop(void* func, ptrdiff_t sp_offset, 
-                         vmprof_get_virtual_ip_t get_virtual_ip);
-
-void vmprof_register_virtual_function(const char* name, void* start, void* end);
-
-
-int vmprof_enable(int fd, long period_usec, int write_header, char* vips,
-				  int vips_len);
-int vmprof_disable(void);
-
-// XXX: this should be part of _vmprof (the CPython extension), not vmprof (the library)
-void vmprof_set_tramp_range(void* start, void* end);
-
-#endif
diff --git a/pypy/module/_vmprof/test/__init__.py b/pypy/module/_vmprof/test/__init__.py
deleted file mode 100644
diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py
deleted file mode 100644
--- a/pypy/module/_vmprof/test/test__vmprof.py
+++ /dev/null
@@ -1,55 +0,0 @@
-
-import tempfile
-from pypy.tool.pytest.objspace import gettestobjspace
-
-class AppTestVMProf(object):
-    def setup_class(cls):
-        cls.space = gettestobjspace(usemodules=['_vmprof', 'struct'])
-        cls.tmpfile = tempfile.NamedTemporaryFile()
-        cls.w_tmpfileno = cls.space.wrap(cls.tmpfile.fileno())
-        cls.w_tmpfilename = cls.space.wrap(cls.tmpfile.name)
-        cls.tmpfile2 = tempfile.NamedTemporaryFile()
-        cls.w_tmpfileno2 = cls.space.wrap(cls.tmpfile2.fileno())
-        cls.w_tmpfilename2 = cls.space.wrap(cls.tmpfile2.name)
-
-    def test_import_vmprof(self):
-        import struct, sys
-
-        WORD = struct.calcsize('l')
-        
-        def count(s):
-            i = 0
-            count = 0
-            i += 5 * WORD # header
-            while i < len(s):
-                assert s[i] == '\x02'
-                i += 1
-                _, size = struct.unpack("ll", s[i:i + 2 * WORD])
-                count += 1
-                i += 2 * WORD + size
-            return count
-        
-        import _vmprof
-        _vmprof.enable(self.tmpfileno)
-        _vmprof.disable()
-        s = open(self.tmpfilename).read()
-        no_of_codes = count(s)
-        assert no_of_codes > 10
-        d = {}
-
-        exec """def foo():
-            pass
-        """ in d
-
-        _vmprof.enable(self.tmpfileno2)
-
-        exec """def foo2():
-            pass
-        """ in d
-
-        _vmprof.disable()
-        s = open(self.tmpfilename2).read()
-        no_of_codes2 = count(s)
-        assert "py:foo:" in s
-        assert "py:foo2:" in s
-        assert no_of_codes2 >= no_of_codes + 2 # some extra codes from tests
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -35,9 +35,6 @@
     name = opcode_method_names[ord(bytecode.co_code[next_instr])]
     return '%s #%d %s' % (bytecode.get_repr(), next_instr, name)
 
-def get_unique_id(next_instr, is_being_profiled, bytecode):
-    return bytecode._unique_id
-
 
 def should_unroll_one_iteration(next_instr, is_being_profiled, bytecode):
     return (bytecode.co_flags & CO_GENERATOR) != 0
@@ -48,7 +45,6 @@
     virtualizables = ['frame']
 
 pypyjitdriver = PyPyJitDriver(get_printable_location = get_printable_location,
-                              get_unique_id = get_unique_id,
                               should_unroll_one_iteration =
                               should_unroll_one_iteration,
                               name='pypyjit')
diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -57,7 +57,6 @@
         BaseAssembler.setup_once(self)
 
     def setup(self, looptoken):
-        BaseAssembler.setup(self, looptoken)
         assert self.memcpy_addr != 0, 'setup_once() not called?'
         if we_are_translated():
             self.debug = False
@@ -72,6 +71,7 @@
         self.mc.datablockwrapper = self.datablockwrapper
         self.target_tokens_currently_compiling = {}
         self.frame_depth_to_patch = []
+        self._finish_gcmap = lltype.nullptr(jitframe.GCMAP)
 
     def teardown(self):
         self.current_clt = None
@@ -659,7 +659,6 @@
             assert len(set(inputargs)) == len(inputargs)
 
         self.setup(original_loop_token)
-        self.codemap.inherit_code_from_position(faildescr.adr_jump_offset)
         descr_number = compute_unique_id(faildescr)
         if log:
             operations = self._inject_debugging_code(faildescr, operations,
@@ -880,12 +879,8 @@
         self.datablockwrapper.done()      # finish using cpu.asmmemmgr
         self.datablockwrapper = None
         allblocks = self.get_asmmemmgr_blocks(looptoken)
-        size = self.mc.get_relative_pos() 
-        res = self.mc.materialize(self.cpu.asmmemmgr, allblocks,
+        return self.mc.materialize(self.cpu.asmmemmgr, allblocks,
                                    self.cpu.gc_ll_descr.gcrootmap)
-        self.cpu.asmmemmgr.register_codemap(
-            self.codemap.get_final_bytecode(res, size))
-        return res
 
     def update_frame_depth(self, frame_depth):
         baseofs = self.cpu.get_baseofs_of_frame_field()
diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -392,9 +392,6 @@
         else:
             self.rm._sync_var(v)
 
-    def prepare_op_debug_merge_point(self, op, fcond):
-        self.assembler.codemap.debug_merge_point(op)
-
     def _prepare_op_int_add(self, op, fcond):
         boxes = op.getarglist()
         a0, a1 = boxes
diff --git a/rpython/jit/backend/llsupport/asmmemmgr.py b/rpython/jit/backend/llsupport/asmmemmgr.py
--- a/rpython/jit/backend/llsupport/asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/asmmemmgr.py
@@ -4,10 +4,7 @@
 from rpython.rlib import rmmap
 from rpython.rlib.debug import debug_start, debug_print, debug_stop
 from rpython.rlib.debug import have_debug_prints
-from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rlib.rbisect import bisect, bisect_tuple
-
-_memmngr = None # global reference so we can use @entrypoint :/
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 
 
 class AsmMemoryManager(object):
@@ -27,12 +24,6 @@
         self.free_blocks = {}      # map {start: stop}
         self.free_blocks_end = {}  # map {stop: start}
         self.blocks_by_size = [[] for i in range(self.num_indices)]
-        # two lists of jit addresses (sorted) and the corresponding stack
-        # depths
-        self.jit_addr_map = []
-        self.jit_frame_depth_map = []
-        self.jit_codemap = []
-        # see codemap.py
 
     def malloc(self, minsize, maxsize):
         """Allocate executable memory, between minsize and maxsize bytes,
@@ -54,13 +45,6 @@
         if r_uint is not None:
             self.total_mallocs -= r_uint(stop - start)
         self._add_free_block(start, stop)
-        # fix up jit_addr_map
-        jit_adr_start = bisect(self.jit_addr_map, start)
-        jit_adr_stop = bisect(self.jit_addr_map, stop)
-        self.jit_addr_map = (self.jit_addr_map[:jit_adr_start] +
-                             self.jit_addr_map[jit_adr_stop:])
-        self.jit_frame_depth_map = (self.jit_frame_depth_map[:jit_adr_start] +
-                                    self.jit_frame_depth_map[jit_adr_stop:])
 
     def open_malloc(self, minsize):
         """Allocate at least minsize bytes.  Returns (start, stop)."""
@@ -167,35 +151,6 @@
         del self.free_blocks_end[stop]
         return (start, stop)
 
-    def register_frame_depth_map(self, rawstart, frame_positions,
-                                 frame_assignments):
-        if not frame_positions:
-            return
-        if not self.jit_addr_map or rawstart > self.jit_addr_map[-1]:
-            start = len(self.jit_addr_map)
-            self.jit_addr_map += [0] * len(frame_positions)
-            self.jit_frame_depth_map += [0] * len(frame_positions)
-        else:
-            start = bisect(self.jit_addr_map, rawstart)
-            self.jit_addr_map = (self.jit_addr_map[:start] +
-                                 [0] * len(frame_positions) +
-                                 self.jit_addr_map[start:])
-            self.jit_frame_depth_map = (self.jit_frame_depth_map[:start] +
-                                 [0] * len(frame_positions) +
-                                 self.jit_frame_depth_map[start:])
-        for i, pos in enumerate(frame_positions):
-            self.jit_addr_map[i + start] = pos + rawstart
-            self.jit_frame_depth_map[i + start] = frame_assignments[i]
-
-    def register_codemap(self, codemap):
-        start = codemap[0]
-        pos = bisect_tuple(self.jit_codemap, start)
-        if pos == len(self.jit_codemap): # common case
-            self.jit_codemap.append(codemap)
-        else:
-            self.jit_codemap = (self.jit_codemap[:pos] + [codemap] +
-                                self.jit_codemap[pos:])
-
     def _delete(self):
         "NOT_RPYTHON"
         if self._allocated:
@@ -257,9 +212,6 @@
 
     gcroot_markers = None
 
-    frame_positions = None
-    frame_assignments = None
-
     def __init__(self, translated=None):
         if translated is None:
             translated = we_are_translated()
@@ -364,10 +316,6 @@
             assert gcrootmap is not None
             for pos, mark in self.gcroot_markers:
                 gcrootmap.register_asm_addr(rawstart + pos, mark)
-        asmmemmgr.register_frame_depth_map(rawstart, self.frame_positions,
-                                           self.frame_assignments)
-        self.frame_positions = None
-        self.frame_assignments = None
         return rawstart
 
     def _become_a_plain_block_builder(self):
diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -1,7 +1,6 @@
 from rpython.jit.backend.llsupport import jitframe
 from rpython.jit.backend.llsupport.memcpy import memcpy_fn, memset_fn
 from rpython.jit.backend.llsupport.symbolic import WORD
-from rpython.jit.backend.llsupport.codemap import CodemapBuilder
 from rpython.jit.metainterp.history import (INT, REF, FLOAT, JitCellToken,
     ConstInt, BoxInt, AbstractFailDescr)
 from rpython.jit.metainterp.resoperation import ResOperation, rop
@@ -129,10 +128,6 @@
                                               track_allocation=False)
         self.gcmap_for_finish[0] = r_uint(1)
 
-    def setup(self, looptoken):
-        self.codemap = CodemapBuilder()
-        self._finish_gcmap = lltype.nullptr(jitframe.GCMAP)
-
     def set_debug(self, v):
         r = self._debug
         self._debug = v
@@ -199,9 +194,6 @@
         guardtok.faildescr.rd_locs = positions
         return fail_descr, target
 
-    def debug_merge_point(self, op):
-        self.codemap.debug_merge_point(op, self.mc.get_relative_pos())
-
     def call_assembler(self, op, guard_op, argloc, vloc, result_loc, tmploc):
         self._store_force_index(guard_op)
         descr = op.getdescr()
@@ -284,9 +276,6 @@
         # YYY very minor leak -- we need the counters to stay alive
         # forever, just because we want to report them at the end
         # of the process
-
-        # XXX the numbers here are ALMOST unique, but not quite, use a counter
-        #     or something
         struct = lltype.malloc(DEBUG_COUNTER, flavor='raw',
                                track_allocation=False)
         struct.i = 0
diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py
deleted file mode 100644
--- a/rpython/jit/backend/llsupport/codemap.py
+++ /dev/null
@@ -1,142 +0,0 @@
-
-""" Bytecode for storage in asmmemmgr.jit_codemap. Format is as follows:
-
- list of tuples of shape (addr, machine code size, bytecode info)
- where bytecode info is a string made up of:
-    8 bytes unique_id, 4 bytes start_addr (relative), 4 bytes size (relative),
-    2 bytes how many items to skip to go to the next on similar level
-    [so far represented by a list of integers for simplicity]
-
-"""
-
-from rpython.rlib import rgc
-from rpython.rlib.entrypoint import jit_entrypoint
-from rpython.jit.backend.llsupport import asmmemmgr
-from rpython.rlib.rbisect import bisect, bisect_tuple
-from rpython.rtyper.lltypesystem import lltype, rffi
-
- at jit_entrypoint([lltype.Signed], lltype.Signed,
-                c_name='pypy_jit_stack_depth_at_loc')
- at rgc.no_collect
-def stack_depth_at_loc(loc):
-    _memmngr = asmmemmgr._memmngr
-
-    pos = bisect(_memmngr.jit_addr_map, loc)
-    if pos == 0 or pos == len(_memmngr.jit_addr_map):
-        return -1
-    return _memmngr.jit_frame_depth_map[pos-1]
-
- at jit_entrypoint([], lltype.Signed, c_name='pypy_jit_start_addr')
-def jit_start_addr():
-    _memmngr = asmmemmgr._memmngr
-
-    return _memmngr.jit_addr_map[0]
-
- at jit_entrypoint([], lltype.Signed, c_name='pypy_jit_end_addr')
-def jit_end_addr():
-    _memmngr = asmmemmgr._memmngr
-
-    return _memmngr.jit_addr_map[-1]
-
- at jit_entrypoint([lltype.Signed], lltype.Signed,
-                c_name='pypy_find_codemap_at_addr')
-def find_codemap_at_addr(addr):
-    _memmngr = asmmemmgr._memmngr
-
-    res = bisect_tuple(_memmngr.jit_codemap, addr) - 1
-    if res == len(_memmngr.jit_codemap):
-        return -1
-    return res
-
- at jit_entrypoint([lltype.Signed, lltype.Signed,
-                 rffi.CArrayPtr(lltype.Signed)], lltype.Signed,
-                 c_name='pypy_yield_codemap_at_addr')
-def yield_bytecode_at_addr(codemap_no, addr, current_pos_addr):
-    """ will return consecutive unique_ids from codemap, starting from position
-    `pos` until addr
-    """
-    _memmngr = asmmemmgr._memmngr
-
-    codemap = _memmngr.jit_codemap[codemap_no]
-    current_pos = current_pos_addr[0]
-    start_addr = codemap[0]
-    rel_addr = addr - start_addr
-    while True:
-        if current_pos >= len(codemap[2]):
-            return 0
-        next_start = codemap[2][current_pos + 1]
-        if next_start > rel_addr:
-            return 0
-        next_stop = codemap[2][current_pos + 2]
-        if next_stop > rel_addr:
-            current_pos_addr[0] = current_pos + 4
-            return codemap[2][current_pos]
-        # we need to skip potentially more than one
-        current_pos = codemap[2][current_pos + 3]
-
-def unpack_traceback(addr):
-    codemap_pos = find_codemap_at_addr(addr)
-    assert codemap_pos >= 0
-    storage = lltype.malloc(rffi.CArray(lltype.Signed), 1, flavor='raw')
-    storage[0] = 0
-    res = []
-    while True:
-        item = yield_bytecode_at_addr(codemap_pos, addr, storage)
-        if item == 0:
-            break
-        res.append(item)
-    lltype.free(storage, flavor='raw')
-    return res
-
-
-class CodemapBuilder(object):
-    def __init__(self):
-        self.l = []
-        self.patch_position = []
-        self.last_call_depth = -1
-
-    def debug_merge_point(self, op, pos):
-        call_depth = op.getarg(1).getint()
-        if call_depth != self.last_call_depth:
-            unique_id = op.getarg(3).getint()
-            if unique_id == 0: # uninteresting case
-                return
-            assert unique_id & 1 == 0
-            if call_depth > self.last_call_depth:
-                self.l.append(unique_id)
-                self.l.append(pos) # <- this is a relative pos
-                self.patch_position.append(len(self.l))
-                self.l.append(0) # marker
-                self.l.append(0) # second marker
-            else:
-                for i in range(self.last_call_depth - call_depth):
-                    to_patch = self.patch_position.pop()
-                    self.l[to_patch] = pos
-                    self.l[to_patch + 1] = len(self.l)
-            self.last_call_depth = call_depth
-
-    def inherit_code_from_position(self, pos):
-        lst = unpack_traceback(pos)
-        self.last_call_depth = len(lst) - 1
-        for item in lst:
-            self.l.append(item)
-            self.l.append(0)
-            self.patch_position.append(len(self.l))
-            self.l.append(0) # marker
-            self.l.append(0) # second marker
-
-    def get_final_bytecode(self, addr, size):
-        while self.patch_position:
-            pos = self.patch_position.pop()
-            self.l[pos] = size
-            self.l[pos + 1] = len(self.l)
-        # at the end there should be no zeros
-        for i in range(len(self.l) / 4):
-            item = self.l[i * 4] # unique_id
-            assert item > 0 # no zeros here
-            item = self.l[i * 4 + 2] # end in asm
-            assert item > 0
-            item = self.l[i * 4 + 3] # end in l
-            assert item > 0
-        return (addr, size, self.l) # XXX compact self.l
-
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -16,7 +16,7 @@
     FieldDescr, ArrayDescr, CallDescr, InteriorFieldDescr,
     FLAG_POINTER, FLAG_FLOAT)
 from rpython.jit.backend.llsupport.memcpy import memset_fn
-from rpython.jit.backend.llsupport import asmmemmgr
+from rpython.jit.backend.llsupport.asmmemmgr import AsmMemoryManager
 from rpython.rlib.unroll import unrolling_iterable
 
 
@@ -48,8 +48,7 @@
             self._setup_exception_handling_translated()
         else:
             self._setup_exception_handling_untranslated()
-        self.asmmemmgr = asmmemmgr.AsmMemoryManager()
-        asmmemmgr._memmngr = self.asmmemmgr
+        self.asmmemmgr = AsmMemoryManager()
         self._setup_frame_realloc(translate_support_code)
         ad = self.gc_ll_descr.getframedescrs(self).arraydescr
         self.signedarraydescr = ad
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -17,6 +17,8 @@
 class GcRewriterAssembler(object):
     """ This class performs the following rewrites on the list of operations:
 
+     - Remove the DEBUG_MERGE_POINTs.
+
      - Turn all NEW_xxx to either a CALL_MALLOC_GC, or a CALL_MALLOC_NURSERY
        followed by SETFIELDs in order to initialize their GC fields.  The
        two advantages of CALL_MALLOC_NURSERY is that it inlines the common
@@ -58,6 +60,8 @@
         #
         for i in range(len(operations)):
             op = operations[i]
+            if op.getopnum() == rop.DEBUG_MERGE_POINT:
+                continue
             # ---------- turn NEWxxx into CALL_MALLOC_xxx ----------
             if op.is_malloc():
                 self.handle_malloc_operation(op)
diff --git a/rpython/jit/backend/llsupport/test/test_asmmemmgr.py b/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
--- a/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
@@ -2,7 +2,6 @@
 from rpython.jit.backend.llsupport.asmmemmgr import AsmMemoryManager
 from rpython.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
 from rpython.jit.backend.llsupport.asmmemmgr import BlockBuilderMixin
-from rpython.jit.backend.llsupport import asmmemmgr
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib import debug
 
@@ -158,7 +157,6 @@
         class FakeGcRootMap:
             def register_asm_addr(self, retaddr, mark):
                 puts.append((retaddr, mark))
-
         #
         mc = BlockBuilderMixin()
         mc.writechar('X')
@@ -268,16 +266,3 @@
     md.done()
     assert allblocks == [(1597, 1697), (1797, 1835)]
     assert ops == [('free', 1835, 1897)]
-
-def test_find_jit_frame_depth():
-    mgr = AsmMemoryManager()
-    mgr.register_frame_depth_map(11, [0, 5, 10], [1, 2, 3])
-    mgr.register_frame_depth_map(30, [0, 5, 10], [4, 5, 6])
-    mgr.register_frame_depth_map(0, [0, 5, 10], [7, 8, 9])
-    asmmemmgr._memmngr = mgr
-    assert asmmemmgr.stack_depth_at_loc(13) == 1
-    assert asmmemmgr.stack_depth_at_loc(-3) == -1
-    assert asmmemmgr.stack_depth_at_loc(41) == -1
-    assert asmmemmgr.stack_depth_at_loc(5) == 8
-    assert asmmemmgr.stack_depth_at_loc(17) == 2
-    assert asmmemmgr.stack_depth_at_loc(38) == 5
diff --git a/rpython/jit/backend/llsupport/test/ztranslation_test.py b/rpython/jit/backend/llsupport/test/ztranslation_test.py
--- a/rpython/jit/backend/llsupport/test/ztranslation_test.py
+++ b/rpython/jit/backend/llsupport/test/ztranslation_test.py
@@ -2,7 +2,7 @@
 from rpython.tool.udir import udir
 from rpython.rlib.jit import JitDriver, unroll_parameters, set_param
 from rpython.rlib.jit import PARAMETERS, dont_look_inside
-from rpython.rlib.jit import promote, _get_virtualizable_token
+from rpython.rlib.jit import promote
 from rpython.rlib import jit_hooks, rposix
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib.rthread import ThreadLocalReference, ThreadLocalField
@@ -28,15 +28,12 @@
         # - floats neg and abs
         # - llexternal with macro=True
 
-        class BasicFrame(object):
+        class Frame(object):
             _virtualizable_ = ['i']
 
             def __init__(self, i):
                 self.i = i
 
-        class Frame(BasicFrame):
-            pass
-
         eci = ExternalCompilationInfo(post_include_bits=['''
 #define pypy_my_fabs(x)  fabs(x)
 '''])
@@ -62,7 +59,6 @@
             while frame.i > 3:
                 jitdriver.can_enter_jit(frame=frame, total=total, j=j)
                 jitdriver.jit_merge_point(frame=frame, total=total, j=j)
-                _get_virtualizable_token(frame)
                 total += frame.i
                 if frame.i >= 20:
                     frame.i -= 2
diff --git a/rpython/jit/backend/x86/arch.py b/rpython/jit/backend/x86/arch.py
--- a/rpython/jit/backend/x86/arch.py
+++ b/rpython/jit/backend/x86/arch.py
@@ -51,6 +51,3 @@
     THREADLOCAL_OFS = (FRAME_FIXED_SIZE - 1) * WORD
 
 assert PASS_ON_MY_FRAME >= 12       # asmgcc needs at least JIT_USE_WORDS + 3
-
-# return address, followed by FRAME_FIXED_SIZE words
-DEFAULT_FRAME_BYTES = (1 + FRAME_FIXED_SIZE) * WORD
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -18,8 +18,7 @@
 from rpython.jit.backend.llsupport.regalloc import (get_scale, valid_addressing_size)
 from rpython.jit.backend.x86.arch import (FRAME_FIXED_SIZE, WORD, IS_X86_64,
                                        JITFRAME_FIXED_SIZE, IS_X86_32,
-                                       PASS_ON_MY_FRAME, THREADLOCAL_OFS,
-                                       DEFAULT_FRAME_BYTES)
+                                       PASS_ON_MY_FRAME, THREADLOCAL_OFS)
 from rpython.jit.backend.x86.regloc import (eax, ecx, edx, ebx, esp, ebp, esi,
     xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, r8, r9, r10, r11, edi,
     r12, r13, r14, r15, X86_64_SCRATCH_REG, X86_64_XMM_SCRATCH_REG,
@@ -67,7 +66,6 @@
             self._build_float_constants()
 
     def setup(self, looptoken):
-        BaseAssembler.setup(self, looptoken)
         assert self.memcpy_addr != 0, "setup_once() not called?"
         self.current_clt = looptoken.compiled_loop_token
         self.pending_guard_tokens = []
@@ -82,6 +80,7 @@
                                                         allblocks)
         self.target_tokens_currently_compiling = {}
         self.frame_depth_to_patch = []
+        self._finish_gcmap = lltype.nullptr(jitframe.GCMAP)
 
     def teardown(self):
         self.pending_guard_tokens = None
@@ -268,10 +267,6 @@
         # the correct "ret" arg
         offset = mc.get_relative_pos() - jz_location
         mc.overwrite32(jz_location-4, offset)
-        # From now on this function is basically "merged" with
-        # its caller and so contains DEFAULT_FRAME_BYTES bytes
-        # plus my own return address, which we'll ignore next
-        mc.force_frame_size(DEFAULT_FRAME_BYTES + WORD)
         mc.ADD_ri(esp.value, WORD)
         mc.JMP(imm(self.propagate_exception_path))
         #
@@ -283,7 +278,6 @@
             return      # not supported (for tests, or non-translated)
         #
         self.mc = codebuf.MachineCodeBlockWrapper()
-        self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         #
         # read and reset the current exception
 
@@ -337,10 +331,7 @@
         offset = mc.get_relative_pos() - jnz_location
         assert 0 < offset <= 127
         mc.overwrite(jnz_location-1, chr(offset))
-        # From now on this function is basically "merged" with
-        # its caller and so contains DEFAULT_FRAME_BYTES bytes
-        # plus my own return address, which we'll ignore next
-        mc.force_frame_size(DEFAULT_FRAME_BYTES + WORD)
+        # adjust the esp to point back to the previous return
         mc.ADD_ri(esp.value, WORD)
         mc.JMP(imm(self.propagate_exception_path))
         #
@@ -418,8 +409,6 @@
                 mc.LEA_rs(esp.value, 2 * WORD)
             self._pop_all_regs_from_frame(mc, [], withfloats, callee_only=True)
             mc.RET16_i(WORD)
-            # Note that wb_slowpath[0..3] end with a RET16_i, which must be
-            # taken care of in the caller by stack_frame_size_delta(-WORD)
         else:
             if IS_X86_32:
                 mc.MOV_rs(edx.value, 4 * WORD)
@@ -525,8 +514,6 @@
             assert len(set(inputargs)) == len(inputargs)
 
         self.setup(original_loop_token)
-        self.codemap.inherit_code_from_position(faildescr.adr_jump_offset)
-        self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         descr_number = compute_unique_id(faildescr)
         if log:
             operations = self._inject_debugging_code(faildescr, operations,
@@ -687,12 +674,8 @@
         self.datablockwrapper.done()      # finish using cpu.asmmemmgr
         self.datablockwrapper = None
         allblocks = self.get_asmmemmgr_blocks(looptoken)
-        size = self.mc.get_relative_pos()
-        res = self.mc.materialize(self.cpu.asmmemmgr, allblocks,
-                                  self.cpu.gc_ll_descr.gcrootmap)
-        self.cpu.asmmemmgr.register_codemap(
-            self.codemap.get_final_bytecode(res, size))
-        return res
+        return self.mc.materialize(self.cpu.asmmemmgr, allblocks,
+                                   self.cpu.gc_ll_descr.gcrootmap)
 
     def patch_jump_for_descr(self, faildescr, adr_new_target):
         adr_jump_offset = faildescr.adr_jump_offset
@@ -703,7 +686,6 @@
         # place, but clobber the recovery stub with a jump to the real
         # target.
         mc = codebuf.MachineCodeBlockWrapper()
-        mc.force_frame_size(DEFAULT_FRAME_BYTES)
         if rx86.fits_in_32bits(offset):
             mc.writeimm32(offset)
             mc.copy_to_raw_memory(adr_jump_offset)
@@ -1774,7 +1756,6 @@
 
     def generate_propagate_error_64(self):
         assert WORD == 8
-        self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         startpos = self.mc.get_relative_pos()
         self.mc.JMP(imm(self.propagate_exception_path))
         return startpos
@@ -1782,7 +1763,6 @@
     def generate_quick_failure(self, guardtok):
         """ Gather information about failure
         """
-        self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         startpos = self.mc.get_relative_pos()
         fail_descr, target = self.store_info_on_descr(startpos, guardtok)
         self.mc.PUSH(imm(fail_descr))
@@ -1858,9 +1838,6 @@
 
     def _build_failure_recovery(self, exc, withfloats=False):
         mc = codebuf.MachineCodeBlockWrapper()
-        # this is jumped to, from a stack that has DEFAULT_FRAME_BYTES
-        # followed by 2 extra words just pushed
-        mc.force_frame_size(DEFAULT_FRAME_BYTES + 2 * WORD)
         self.mc = mc
 
         self._push_all_regs_to_frame(mc, [], withfloats)
@@ -1932,7 +1909,6 @@
             self.mc.J_il(rx86.Conditions[condition], 0)
         else:
             self.mc.JMP_l(0)
-            self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         guard_token.pos_jump_offset = self.mc.get_relative_pos() - 4
         self.pending_guard_tokens.append(guard_token)
 
@@ -2044,7 +2020,6 @@
         offset = jmp_location - je_location
         assert 0 < offset <= 127
         self.mc.overwrite(je_location - 1, chr(offset))
-        self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         #
         return jmp_location
 
@@ -2129,8 +2104,6 @@
         if is_frame and align_stack:
             mc.SUB_ri(esp.value, 16 - WORD) # erase the return address
         mc.CALL(imm(self.wb_slowpath[helper_num]))
-        if not is_frame:
-            mc.stack_frame_size_delta(-WORD)
         if is_frame and align_stack:
             mc.ADD_ri(esp.value, 16 - WORD) # erase the return address
 
@@ -2367,7 +2340,6 @@
         offset = self.mc.get_relative_pos() - jmp_adr1
         assert 0 < offset <= 127
         self.mc.overwrite(jmp_adr1-1, chr(offset))
-        self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         # write down the tid, but not if it's the result of the CALL
         self.mc.MOV(mem(eax, 0), imm(arraydescr.tid))
         # while we're at it, this line is not needed if we've done the CALL
diff --git a/rpython/jit/backend/x86/callbuilder.py b/rpython/jit/backend/x86/callbuilder.py
--- a/rpython/jit/backend/x86/callbuilder.py
+++ b/rpython/jit/backend/x86/callbuilder.py
@@ -58,7 +58,7 @@
             self.fnloc_is_immediate = False
             self.fnloc = None
             self.arglocs = arglocs + [fnloc]
-        self.start_frame_size = self.mc._frame_size
+        self.current_esp = 0     # 0 or (usually) negative, counted in bytes
 
     def select_call_release_gil_mode(self):
         AbstractCallBuilder.select_call_release_gil_mode(self)
@@ -70,15 +70,13 @@
     def subtract_esp_aligned(self, count):
         if count > 0:
             align = align_stack_words(count)
+            self.current_esp -= align * WORD
             self.mc.SUB_ri(esp.value, align * WORD)
 
-    def get_current_esp(self):
-        return self.start_frame_size - self.mc._frame_size
-
     def restore_stack_pointer(self, target_esp=0):
-        current_esp = self.get_current_esp()
-        if current_esp != target_esp:
-            self.mc.ADD_ri(esp.value, target_esp - current_esp)
+        if self.current_esp != target_esp:
+            self.mc.ADD_ri(esp.value, target_esp - self.current_esp)
+            self.current_esp = target_esp
 
     def load_result(self):
         """Overridden in CallBuilder32 and CallBuilder64"""
@@ -101,10 +99,9 @@
         # after the rearrangements done just before, ignoring the return
         # value eax, if necessary
         assert not self.is_call_release_gil
-        current_esp = self.get_current_esp()
-        self.change_extra_stack_depth = (current_esp != 0)
+        self.change_extra_stack_depth = (self.current_esp != 0)
         if self.change_extra_stack_depth:
-            self.asm.set_extra_stack_depth(self.mc, -current_esp)
+            self.asm.set_extra_stack_depth(self.mc, -self.current_esp)
         noregs = self.asm.cpu.gc_ll_descr.is_shadow_stack()
         gcmap = self.asm._regalloc.get_gcmap([eax], noregs=noregs)
         self.asm.push_gcmap(self.mc, gcmap, store=True)
@@ -145,7 +142,7 @@
             # and 5/7 words as described for asmgcroot.ASM_FRAMEDATA, for a
             # total size of JIT_USE_WORDS.  This structure is found at
             # [ESP+css].
-            css = -self.get_current_esp() + (
+            css = -self.current_esp + (
                 WORD * (PASS_ON_MY_FRAME - asmgcroot.JIT_USE_WORDS))
             assert css >= 2 * WORD
             # Save ebp
@@ -181,7 +178,7 @@
             else:
                 self.tlofs_reg = r12
             self.mc.MOV_rs(self.tlofs_reg.value,
-                           THREADLOCAL_OFS - self.get_current_esp())
+                           THREADLOCAL_OFS - self.current_esp)
             if self.asm._is_asmgcc():
                 self.mc.AND_ri(self.tlofs_reg.value, ~1)
         return self.tlofs_reg
@@ -454,10 +451,7 @@
         else:
             self.mc.CALL(self.fnloc)
             if self.callconv != FFI_DEFAULT_ABI:
-                # in the STDCALL ABI, the CALL above has an effect on
-                # the stack depth.  Adjust 'mc._frame_size'.
-                delta = self._fix_stdcall(self.callconv)
-                self.mc.stack_frame_size_delta(-delta)
+                self.current_esp += self._fix_stdcall(self.callconv)
 
     def _fix_stdcall(self, callconv):
         from rpython.rlib.clibffi import FFI_STDCALL
diff --git a/rpython/jit/backend/x86/codebuf.py b/rpython/jit/backend/x86/codebuf.py
--- a/rpython/jit/backend/x86/codebuf.py
+++ b/rpython/jit/backend/x86/codebuf.py
@@ -23,7 +23,6 @@
                               codebuilder_cls):
     def __init__(self):
         self.init_block_builder()
-        codebuilder_cls.__init__(self)
         # a list of relative positions; for each position p, the bytes
         # at [p-4:p] encode an absolute address that will need to be
         # made relative.  Only works on 32-bit!
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -12,7 +12,7 @@
      valid_addressing_size)
 from rpython.jit.backend.x86 import rx86
 from rpython.jit.backend.x86.arch import (WORD, JITFRAME_FIXED_SIZE, IS_X86_32,
-    IS_X86_64, DEFAULT_FRAME_BYTES)
+    IS_X86_64)
 from rpython.jit.backend.x86.jump import remap_frame_layout_mixed
 from rpython.jit.backend.x86.regloc import (FrameLoc, RegLoc, ConstFloatLoc,
     FloatImmedLoc, ImmedLoc, imm, imm0, imm1, ecx, eax, edx, ebx, esi, edi,
@@ -314,7 +314,6 @@
         while i < len(operations):
             op = operations[i]
             self.assembler.mc.mark_op(op)
-            assert self.assembler.mc._frame_size == DEFAULT_FRAME_BYTES
             self.rm.position = i
             self.xrm.position = i
             if op.has_no_side_effect() and op.result not in self.longevity:
@@ -1324,7 +1323,7 @@
         assembler.closing_jump(self.jump_target_descr)
 
     def consider_debug_merge_point(self, op):
-        self.assembler.debug_merge_point(op)
+        pass
 
     def consider_jit_debug(self, op):
         pass
diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py
--- a/rpython/jit/backend/x86/rx86.py
+++ b/rpython/jit/backend/x86/rx86.py
@@ -454,11 +454,6 @@
 class AbstractX86CodeBuilder(object):
     """Abstract base class."""
 
-    def __init__(self):
-        self.frame_positions = []
-        self.frame_assignments = []
-        self.force_frame_size(self.WORD)
-
     def writechar(self, char):
         raise NotImplementedError
 
@@ -476,23 +471,6 @@
         self.writechar(chr((imm >> 16) & 0xFF))
         self.writechar(chr((imm >> 24) & 0xFF))
 
-    def force_frame_size(self, frame_size):
-        self.frame_positions.append(self.get_relative_pos())
-        self.frame_assignments.append(frame_size)
-        self._frame_size = frame_size
-
-    def stack_frame_size_delta(self, delta):
-        "Called when we generate an instruction that changes the value of ESP"
-        self._frame_size += delta
-        self.frame_positions.append(self.get_relative_pos()) 
-        self.frame_assignments.append(self._frame_size)
-        assert self._frame_size >= self.WORD
-
-    def check_stack_size_at_ret(self):
-        assert self._frame_size == self.WORD
-        if not we_are_translated():
-            self._frame_size = None
-
     # ------------------------------ MOV ------------------------------
 
     MOV_ri = insn(register(1), '\xB8', immediate(2))
@@ -503,24 +481,14 @@
     INC_m = insn(rex_w, '\xFF', orbyte(0), mem_reg_plus_const(1))
     INC_j = insn(rex_w, '\xFF', orbyte(0), abs_(1))
 
-    AD1_ri,ADD_rr,ADD_rb,_,_,ADD_rm,ADD_rj,_,_ = common_modes(0)
+    ADD_ri,ADD_rr,ADD_rb,_,_,ADD_rm,ADD_rj,_,_ = common_modes(0)
     OR_ri, OR_rr, OR_rb, _,_,OR_rm, OR_rj, _,_ = common_modes(1)
     AND_ri,AND_rr,AND_rb,_,_,AND_rm,AND_rj,_,_ = common_modes(4)
-    SU1_ri,SUB_rr,SUB_rb,_,_,SUB_rm,SUB_rj,SUB_ji8,SUB_mi8 = common_modes(5)
+    SUB_ri,SUB_rr,SUB_rb,_,_,SUB_rm,SUB_rj,SUB_ji8,SUB_mi8 = common_modes(5)
     SBB_ri,SBB_rr,SBB_rb,_,_,SBB_rm,SBB_rj,_,_ = common_modes(3)
     XOR_ri,XOR_rr,XOR_rb,_,_,XOR_rm,XOR_rj,_,_ = common_modes(6)
     CMP_ri,CMP_rr,CMP_rb,CMP_bi,CMP_br,CMP_rm,CMP_rj,_,_ = common_modes(7)
 
-    def ADD_ri(self, reg, immed):
-        self.AD1_ri(reg, immed)
-        if reg == R.esp:
-            self.stack_frame_size_delta(-immed)
-
-    def SUB_ri(self, reg, immed):
-        self.SU1_ri(reg, immed)
-        if reg == R.esp:
-            self.stack_frame_size_delta(+immed)
-
     CMP_mi8 = insn(rex_w, '\x83', orbyte(7<<3), mem_reg_plus_const(1), immediate(2, 'b'))
     CMP_mi32 = insn(rex_w, '\x81', orbyte(7<<3), mem_reg_plus_const(1), immediate(2))
     CMP_mi = select_8_or_32_bit_immed(CMP_mi8, CMP_mi32)
@@ -570,65 +538,30 @@
     # ------------------------------ Misc stuff ------------------------------
 
     NOP = insn('\x90')
-    RE1 = insn('\xC3')
-    RE116_i = insn('\xC2', immediate(1, 'h'))
+    RET = insn('\xC3')
+    RET16_i = insn('\xC2', immediate(1, 'h'))
 
-    def RET(self):
-        self.check_stack_size_at_ret()
-        self.RE1()
+    PUSH_r = insn(rex_nw, register(1), '\x50')
+    PUSH_b = insn(rex_nw, '\xFF', orbyte(6<<3), stack_bp(1))
+    PUSH_m = insn(rex_nw, '\xFF', orbyte(6<<3), mem_reg_plus_const(1))
+    PUSH_i8 = insn('\x6A', immediate(1, 'b'))
+    PUSH_i32 = insn('\x68', immediate(1, 'i'))
+    def PUSH_i(mc, immed):
+        if single_byte(immed):
+            mc.PUSH_i8(immed)
+        else:
+            mc.PUSH_i32(immed)
 
-    def RET16_i(self, immed):
-        self.check_stack_size_at_ret()

From noreply at buildbot.pypy.org  Fri Apr 17 10:03:03 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 17 Apr 2015 10:03:03 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof2: Backed out changeset a69d4a5a9638
Message-ID: <20150417080303.1EC1A1C1331@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof2
Changeset: r76813:4b972671f5e2
Date: 2015-04-17 10:02 +0200
http://bitbucket.org/pypy/pypy/changeset/4b972671f5e2/

Log:	Backed out changeset a69d4a5a9638

diff too long, truncating to 2000 out of 2680 lines

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -420,3 +420,10 @@
 the terms of the GPL license version 2 or any later version.  Thus the
 gdbm module, provided in the file lib_pypy/gdbm.py, is redistributed
 under the terms of the GPL license as well.
+
+License for 'pypy/module/_vmprof/src'
+--------------------------------------
+
+The code is based on gperftools. You may see a copy of the License for it at
+
+    https://code.google.com/p/gperftools/source/browse/COPYING
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -38,6 +38,9 @@
     "_csv", "cppyy", "_pypyjson"
 ])
 
+if sys.platform.startswith('linux') and sys.maxint > 2147483647:
+    working_modules.add('_vmprof')
+
 translation_modules = default_modules.copy()
 translation_modules.update([
     "fcntl", "time", "select", "signal", "_rawffi", "zlib", "struct", "_md5",
@@ -99,6 +102,7 @@
     "_hashlib"  : ["pypy.module._ssl.interp_ssl"],
     "_minimal_curses": ["pypy.module._minimal_curses.fficurses"],
     "_continuation": ["rpython.rlib.rstacklet"],
+    "_vmprof" : ["pypy.module._vmprof.interp_vmprof"],
     }
 
 def get_module_validator(modname):
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -9,6 +9,7 @@
 from rpython.rlib.signature import signature
 from rpython.rlib.rarithmetic import r_uint, SHRT_MIN, SHRT_MAX, \
     INT_MIN, INT_MAX, UINT_MAX, USHRT_MAX
+from rpython.rlib.rweaklist import RWeakListMixin
 
 from pypy.interpreter.executioncontext import (ExecutionContext, ActionFlag,
     UserDelAction)
@@ -366,6 +367,10 @@
 
 # ____________________________________________________________
 
+class CodeObjWeakList(RWeakListMixin):
+    def __init__(self):
+        self.initialize()
+
 class ObjSpace(object):
     """Base class for the interpreter-level implementations of object spaces.
     http://pypy.readthedocs.org/en/latest/objspace.html"""
@@ -389,6 +394,7 @@
         self.check_signal_action = None   # changed by the signal module
         self.user_del_action = UserDelAction(self)
         self._code_of_sys_exc_info = None
+        self.all_code_objs = CodeObjWeakList()
 
         # can be overridden to a subclass
         self.initialize()
@@ -666,6 +672,16 @@
             assert ec is not None
             return ec
 
+    def register_code_object(self, pycode):
+        callback = self.getexecutioncontext().register_code_callback
+        if callback is not None:
+            callback(self, pycode)
+        self.all_code_objs.add_handle(pycode)
+
+    def set_code_callback(self, callback):
+        ec = self.getexecutioncontext()
+        ec.register_code_callback = callback
+        
     def _freeze_(self):
         return True
 
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -33,6 +33,11 @@
         self.profilefunc = None
         self.w_profilefuncarg = None
         self.thread_disappeared = False   # might be set to True after os.fork()
+        self.register_code_callback = None
+        if sys.maxint == 2147483647:
+            self._code_unique_id = 0 # XXX this is wrong, it won't work on 32bit
+        else:
+            self._code_unique_id = 0x7000000000000000
 
     @staticmethod
     def _mark_thread_disappeared(space):
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -14,9 +14,10 @@
     CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS, CO_NESTED,
     CO_GENERATOR, CO_KILL_DOCSTRING, CO_YIELD_INSIDE_TRY)
 from pypy.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT
-from rpython.rlib.rarithmetic import intmask
+from rpython.rlib.rarithmetic import intmask, r_longlong
 from rpython.rlib.objectmodel import compute_hash
 from rpython.rlib import jit
+from rpython.rlib.debug import debug_start, debug_stop, debug_print
 
 
 class BytecodeCorruption(Exception):
@@ -54,8 +55,9 @@
     "CPython-style code objects."
     _immutable_ = True
     _immutable_fields_ = ["co_consts_w[*]", "co_names_w[*]", "co_varnames[*]",
-                          "co_freevars[*]", "co_cellvars[*]", "_args_as_cellvars[*]"]
-
+                          "co_freevars[*]", "co_cellvars[*]",
+                          "_args_as_cellvars[*]"]
+    
     def __init__(self, space,  argcount, nlocals, stacksize, flags,
                      code, consts, names, varnames, filename,
                      name, firstlineno, lnotab, freevars, cellvars,
@@ -83,6 +85,7 @@
         self.magic = magic
         self._signature = cpython_code_signature(self)
         self._initialize()
+        space.register_code_object(self)
 
     def _initialize(self):
         if self.co_cellvars:
@@ -124,6 +127,15 @@
             from pypy.objspace.std.mapdict import init_mapdict_cache
             init_mapdict_cache(self)
 
+        ec = self.space.getexecutioncontext()
+        self._unique_id = ec._code_unique_id
+        ec._code_unique_id += 2 # so we have one bit that we can mark stuff
+        # with
+
+    def _get_full_name(self):
+        return "py:%s:%d:%s" % (self.co_name, self.co_firstlineno,
+                                self.co_filename)
+
     def _cleanup_(self):
         if (self.magic == cpython_magic and
             '__pypy__' not in sys.builtin_module_names):
diff --git a/pypy/module/_vmprof/__init__.py b/pypy/module/_vmprof/__init__.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/__init__.py
@@ -0,0 +1,18 @@
+from pypy.interpreter.mixedmodule import MixedModule
+
+class Module(MixedModule):
+    """
+    Write me :)
+    """
+    appleveldefs = {
+    }
+
+    interpleveldefs = {
+        'enable': 'interp_vmprof.enable',
+        'disable': 'interp_vmprof.disable',
+    }
+
+    def setup_after_space_initialization(self):
+        # force the __extend__ hacks to occur early
+        from pypy.module._vmprof.interp_vmprof import VMProf
+        self.vmprof = VMProf()
diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -0,0 +1,223 @@
+import py, os, sys
+from rpython.rtyper.lltypesystem import lltype, rffi, llmemory
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+from rpython.rtyper.annlowlevel import cast_instance_to_gcref, cast_base_ptr_to_instance
+from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib import jit, rposix, entrypoint
+from rpython.rtyper.tool import rffi_platform as platform
+from rpython.rlib.rstring import StringBuilder
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.error import oefmt, wrap_oserror, OperationError
+from pypy.interpreter.gateway import unwrap_spec
+from pypy.interpreter.pyframe import PyFrame
+
+ROOT = py.path.local(__file__).join('..')
+SRC = ROOT.join('src')
+
+# by default, we statically link vmprof.c into pypy; however, if you set
+# DYNAMIC_VMPROF to True, it will be dynamically linked to the libvmprof.so
+# which is expected to be inside pypy/module/_vmprof/src: this is very useful
+# during development. Note that you have to manually build libvmprof by
+# running make inside the src dir
+DYNAMIC_VMPROF = False
+
+eci_kwds = dict(
+    include_dirs = [SRC],
+    includes = ['vmprof.h', 'trampoline.h'],
+    separate_module_files = [SRC.join('trampoline.asmgcc.s')],
+    libraries = ['unwind'],
+    
+    post_include_bits=["""
+        void* pypy_vmprof_get_virtual_ip(void*);
+        void pypy_vmprof_init(void);
+    """],
+    
+    separate_module_sources=["""
+        void pypy_vmprof_init(void) {
+            vmprof_set_mainloop(pypy_execute_frame_trampoline, 0,
+                                pypy_vmprof_get_virtual_ip);
+        }
+    """],
+    )
+
+
+if DYNAMIC_VMPROF:
+    eci_kwds['libraries'] += ['vmprof']
+    eci_kwds['link_extra'] = ['-Wl,-rpath,%s' % SRC, '-L%s' % SRC]
+else:
+    eci_kwds['separate_module_files'] += [SRC.join('vmprof.c')]
+
+eci = ExternalCompilationInfo(**eci_kwds)
+
+check_eci = eci.merge(ExternalCompilationInfo(separate_module_files=[
+    SRC.join('fake_pypy_api.c')]))
+
+platform.verify_eci(check_eci)
+
+pypy_execute_frame_trampoline = rffi.llexternal(
+    "pypy_execute_frame_trampoline",
+    [llmemory.GCREF, llmemory.GCREF, llmemory.GCREF],
+    llmemory.GCREF,
+    compilation_info=eci,
+    _nowrapper=True, sandboxsafe=True,
+    random_effects_on_gcobjs=True)
+
+pypy_vmprof_init = rffi.llexternal("pypy_vmprof_init", [], lltype.Void,
+                                   compilation_info=eci)
+vmprof_enable = rffi.llexternal("vmprof_enable",
+                                [rffi.INT, rffi.LONG, rffi.INT,
+                                 rffi.CCHARP, rffi.INT],
+                                rffi.INT, compilation_info=eci,
+                                save_err=rffi.RFFI_SAVE_ERRNO)
+vmprof_disable = rffi.llexternal("vmprof_disable", [], rffi.INT,
+                                 compilation_info=eci,
+                                save_err=rffi.RFFI_SAVE_ERRNO)
+
+vmprof_register_virtual_function = rffi.llexternal(
+    "vmprof_register_virtual_function",
+    [rffi.CCHARP, rffi.VOIDP, rffi.VOIDP], lltype.Void,
+    compilation_info=eci, _nowrapper=True)
+
+original_execute_frame = PyFrame.execute_frame.im_func
+original_execute_frame.c_name = 'pypy_pyframe_execute_frame'
+original_execute_frame._dont_inline_ = True
+
+class __extend__(PyFrame):
+    def execute_frame(frame, w_inputvalue=None, operr=None):
+        # go through the asm trampoline ONLY if we are translated but not being JITted.
+        #
+        # If we are not translated, we obviously don't want to go through the
+        # trampoline because there is no C function it can call.
+        #
+        # If we are being JITted, we want to skip the trampoline, else the JIT
+        # cannot see throug it
+        if we_are_translated() and not jit.we_are_jitted():
+            # if we are translated, call the trampoline
+            gc_frame = cast_instance_to_gcref(frame)
+            gc_inputvalue = cast_instance_to_gcref(w_inputvalue)
+            gc_operr = cast_instance_to_gcref(operr)
+            gc_result = pypy_execute_frame_trampoline(gc_frame, gc_inputvalue, gc_operr)
+            return cast_base_ptr_to_instance(W_Root, gc_result)
+        else:
+            return original_execute_frame(frame, w_inputvalue, operr)
+
+
+ at entrypoint.entrypoint_lowlevel('main', [llmemory.GCREF],
+                                'pypy_vmprof_get_virtual_ip', True)
+def get_virtual_ip(gc_frame):
+    frame = cast_base_ptr_to_instance(PyFrame, gc_frame)
+    if jit._get_virtualizable_token(frame):
+        return rffi.cast(rffi.VOIDP, 0)
+    virtual_ip = do_get_virtual_ip(frame)
+    return rffi.cast(rffi.VOIDP, virtual_ip)
+
+def do_get_virtual_ip(frame):
+    return frame.pycode._unique_id
+
+def write_long_to_string_builder(l, b):
+    if sys.maxint == 2147483647:
+        b.append(chr(l & 0xff))
+        b.append(chr((l >> 8) & 0xff))
+        b.append(chr((l >> 16) & 0xff))
+        b.append(chr((l >> 24) & 0xff))
+    else:
+        b.append(chr(l & 0xff))
+        b.append(chr((l >> 8) & 0xff))
+        b.append(chr((l >> 16) & 0xff))
+        b.append(chr((l >> 24) & 0xff))
+        b.append(chr((l >> 32) & 0xff))
+        b.append(chr((l >> 40) & 0xff))
+        b.append(chr((l >> 48) & 0xff))
+        b.append(chr((l >> 56) & 0xff))
+
+class VMProf(object):
+    def __init__(self):
+        self.is_enabled = False
+        self.ever_enabled = False
+        self.mapping_so_far = [] # stored mapping in between runs
+        self.fileno = -1
+
+    def enable(self, space, fileno, period):
+        if self.is_enabled:
+            raise oefmt(space.w_ValueError, "_vmprof already enabled")
+        self.fileno = fileno
+        self.is_enabled = True
+        self.write_header(fileno, period)
+        if not self.ever_enabled:
+            if we_are_translated():
+                pypy_vmprof_init()
+            self.ever_enabled = True
+        for weakcode in space.all_code_objs.get_all_handles():
+            code = weakcode()
+            if code:
+                self.register_code(space, code)
+        space.set_code_callback(vmprof_register_code)
+        if we_are_translated():
+            # does not work untranslated
+            res = vmprof_enable(fileno, period, 0,
+                                lltype.nullptr(rffi.CCHARP.TO), 0)
+        else:
+            res = 0
+        if res == -1:
+            raise wrap_oserror(space, OSError(rposix.get_saved_errno(),
+                                              "_vmprof.enable"))
+
+    def write_header(self, fileno, period):
+        if period == -1:
+            period_usec = 1000000 / 100 #  100hz
+        else:
+            period_usec = period
+        b = StringBuilder()
+        write_long_to_string_builder(0, b)
+        write_long_to_string_builder(3, b)
+        write_long_to_string_builder(0, b)
+        write_long_to_string_builder(period_usec, b)
+        write_long_to_string_builder(0, b)
+        os.write(fileno, b.build())
+
+    def register_code(self, space, code):
+        if self.fileno == -1:
+            raise OperationError(space.w_RuntimeError,
+                                 space.wrap("vmprof not running"))
+        name = code._get_full_name()
+        b = StringBuilder()
+        b.append('\x02')
+        write_long_to_string_builder(code._unique_id, b)
+        write_long_to_string_builder(len(name), b)
+        b.append(name)
+        os.write(self.fileno, b.build())
+
+    def disable(self, space):
+        if not self.is_enabled:
+            raise oefmt(space.w_ValueError, "_vmprof not enabled")
+        self.is_enabled = False
+        self.fileno = -1
+        if we_are_translated():
+           # does not work untranslated
+            res = vmprof_disable()
+        else:
+            res = 0
+        space.set_code_callback(None)
+        if res == -1:
+            raise wrap_oserror(space, OSError(rposix.get_saved_errno(),
+                                              "_vmprof.disable"))
+
+def vmprof_register_code(space, code):
+    from pypy.module._vmprof import Module
+    mod_vmprof = space.getbuiltinmodule('_vmprof')
+    assert isinstance(mod_vmprof, Module)
+    mod_vmprof.vmprof.register_code(space, code)
+        
+ at unwrap_spec(fileno=int, period=int)
+def enable(space, fileno, period=-1):
+    from pypy.module._vmprof import Module
+    mod_vmprof = space.getbuiltinmodule('_vmprof')
+    assert isinstance(mod_vmprof, Module)
+    mod_vmprof.vmprof.enable(space, fileno, period)
+
+def disable(space):
+    from pypy.module._vmprof import Module
+    mod_vmprof = space.getbuiltinmodule('_vmprof')
+    assert isinstance(mod_vmprof, Module)
+    mod_vmprof.vmprof.disable(space)
+
diff --git a/pypy/module/_vmprof/src/config.h b/pypy/module/_vmprof/src/config.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/src/config.h
@@ -0,0 +1,2 @@
+#define HAVE_SYS_UCONTEXT_H
+#define PC_FROM_UCONTEXT uc_mcontext.gregs[REG_RIP]
diff --git a/pypy/module/_vmprof/src/fake_pypy_api.c b/pypy/module/_vmprof/src/fake_pypy_api.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/src/fake_pypy_api.c
@@ -0,0 +1,29 @@
+
+long pypy_jit_start_addr(void)
+{
+	return 3;
+}
+
+long pypy_jit_end_addr(void)
+{
+	return 3;
+}
+
+long pypy_jit_stack_depth_at_loc(long x)
+{
+	return 0;
+}
+
+long pypy_find_codemap_at_addr(long x)
+{
+	return 0;
+}
+
+long pypy_yield_codemap_at_addr(long x, long y, long *a)
+{
+	return 0;
+}
+
+void pypy_pyframe_execute_frame(void)
+{
+}
diff --git a/pypy/module/_vmprof/src/get_custom_offset.c b/pypy/module/_vmprof/src/get_custom_offset.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/src/get_custom_offset.c
@@ -0,0 +1,46 @@
+
+long pypy_jit_start_addr();
+long pypy_jit_end_addr();
+long pypy_jit_stack_depth_at_loc(long);
+long pypy_find_codemap_at_addr(long);
+long pypy_yield_codemap_at_addr(long, long, long*);
+
+void vmprof_set_tramp_range(void* start, void* end)
+{
+}
+
+static ptrdiff_t vmprof_unw_get_custom_offset(void* ip, unw_cursor_t *cp) {
+	intptr_t ip_l = (intptr_t)ip;
+
+	if (ip_l < pypy_jit_start_addr() || ip_l > pypy_jit_end_addr()) {
+		return -1;
+	}
+	return (void*)pypy_jit_stack_depth_at_loc(ip_l);
+}
+
+static long vmprof_write_header_for_jit_addr(void **result, long n,
+											 void *ip, int max_depth)
+{
+	long codemap_pos;
+	long current_pos = 0;
+	intptr_t id;
+	intptr_t addr = (intptr_t)ip;
+
+	if (addr < pypy_jit_start_addr() || addr > pypy_jit_end_addr()) {
+		return n;
+	}
+	codemap_pos = pypy_find_codemap_at_addr(addr);
+	if (codemap_pos == -1) {
+		return n;
+	}
+	while (1) {
+		id = pypy_yield_codemap_at_addr(codemap_pos, addr, ¤t_pos);
+		if (id == 0) {
+			return n;
+		}
+		result[n++] = id;
+		if (n >= max_depth) {
+			return n;
+		}
+	}
+}
diff --git a/pypy/module/_vmprof/src/getpc.h b/pypy/module/_vmprof/src/getpc.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/src/getpc.h
@@ -0,0 +1,187 @@
+// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*-
+// Copyright (c) 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// ---
+// Author: Craig Silverstein
+//
+// This is an internal header file used by profiler.cc.  It defines
+// the single (inline) function GetPC.  GetPC is used in a signal
+// handler to figure out the instruction that was being executed when
+// the signal-handler was triggered.
+//
+// To get this, we use the ucontext_t argument to the signal-handler
+// callback, which holds the full context of what was going on when
+// the signal triggered.  How to get from a ucontext_t to a Program
+// Counter is OS-dependent.
+
+#ifndef BASE_GETPC_H_
+#define BASE_GETPC_H_
+
+#include "config.h"
+
+// On many linux systems, we may need _GNU_SOURCE to get access to
+// the defined constants that define the register we want to see (eg
+// REG_EIP).  Note this #define must come first!
+#define _GNU_SOURCE 1
+// If #define _GNU_SOURCE causes problems, this might work instead.
+// It will cause problems for FreeBSD though!, because it turns off
+// the needed __BSD_VISIBLE.
+//#define _XOPEN_SOURCE 500
+
+#include          // for memcmp
+#if defined(HAVE_SYS_UCONTEXT_H)
+#include 
+#elif defined(HAVE_UCONTEXT_H)
+#include        // for ucontext_t (and also mcontext_t)
+#elif defined(HAVE_CYGWIN_SIGNAL_H)
+#include 
+typedef ucontext ucontext_t;
+#endif
+
+
+// Take the example where function Foo() calls function Bar().  For
+// many architectures, Bar() is responsible for setting up and tearing
+// down its own stack frame.  In that case, it's possible for the
+// interrupt to happen when execution is in Bar(), but the stack frame
+// is not properly set up (either before it's done being set up, or
+// after it's been torn down but before Bar() returns).  In those
+// cases, the stack trace cannot see the caller function anymore.
+//
+// GetPC can try to identify this situation, on architectures where it
+// might occur, and unwind the current function call in that case to
+// avoid false edges in the profile graph (that is, edges that appear
+// to show a call skipping over a function).  To do this, we hard-code
+// in the asm instructions we might see when setting up or tearing
+// down a stack frame.
+//
+// This is difficult to get right: the instructions depend on the
+// processor, the compiler ABI, and even the optimization level.  This
+// is a best effort patch -- if we fail to detect such a situation, or
+// mess up the PC, nothing happens; the returned PC is not used for
+// any further processing.
+struct CallUnrollInfo {
+  // Offset from (e)ip register where this instruction sequence
+  // should be matched. Interpreted as bytes. Offset 0 is the next
+  // instruction to execute. Be extra careful with negative offsets in
+  // architectures of variable instruction length (like x86) - it is
+  // not that easy as taking an offset to step one instruction back!
+  int pc_offset;
+  // The actual instruction bytes. Feel free to make it larger if you
+  // need a longer sequence.
+  unsigned char ins[16];
+  // How many bytes to match from ins array?
+  int ins_size;
+  // The offset from the stack pointer (e)sp where to look for the
+  // call return address. Interpreted as bytes.
+  int return_sp_offset;
+};
+
+
+// The dereferences needed to get the PC from a struct ucontext were
+// determined at configure time, and stored in the macro
+// PC_FROM_UCONTEXT in config.h.  The only thing we need to do here,
+// then, is to do the magic call-unrolling for systems that support it.
+
+// -- Special case 1: linux x86, for which we have CallUnrollInfo
+#if defined(__linux) && defined(__i386) && defined(__GNUC__)
+static const CallUnrollInfo callunrollinfo[] = {
+  // Entry to a function:  push %ebp;  mov  %esp,%ebp
+  // Top-of-stack contains the caller IP.
+  { 0,
+    {0x55, 0x89, 0xe5}, 3,
+    0
+  },
+  // Entry to a function, second instruction:  push %ebp;  mov  %esp,%ebp
+  // Top-of-stack contains the old frame, caller IP is +4.
+  { -1,
+    {0x55, 0x89, 0xe5}, 3,
+    4
+  },
+  // Return from a function: RET.
+  // Top-of-stack contains the caller IP.
+  { 0,
+    {0xc3}, 1,
+    0
+  }
+};
+
+inline void* GetPC(ucontext_t *signal_ucontext) {
+  // See comment above struct CallUnrollInfo.  Only try instruction
+  // flow matching if both eip and esp looks reasonable.
+  const int eip = signal_ucontext->uc_mcontext.gregs[REG_EIP];
+  const int esp = signal_ucontext->uc_mcontext.gregs[REG_ESP];
+  if ((eip & 0xffff0000) != 0 && (~eip & 0xffff0000) != 0 &&
+      (esp & 0xffff0000) != 0) {
+    char* eip_char = reinterpret_cast(eip);
+    for (int i = 0; i < sizeof(callunrollinfo)/sizeof(*callunrollinfo); ++i) {
+      if (!memcmp(eip_char + callunrollinfo[i].pc_offset,
+                  callunrollinfo[i].ins, callunrollinfo[i].ins_size)) {
+        // We have a match.
+        void **retaddr = (void**)(esp + callunrollinfo[i].return_sp_offset);
+        return *retaddr;
+      }
+    }
+  }
+  return (void*)eip;
+}
+
+// Special case #2: Windows, which has to do something totally different.
+#elif defined(_WIN32) || defined(__CYGWIN__) || defined(__CYGWIN32__) || defined(__MINGW32__)
+// If this is ever implemented, probably the way to do it is to have
+// profiler.cc use a high-precision timer via timeSetEvent:
+//    http://msdn2.microsoft.com/en-us/library/ms712713.aspx
+// We'd use it in mode TIME_CALLBACK_FUNCTION/TIME_PERIODIC.
+// The callback function would be something like prof_handler, but
+// alas the arguments are different: no ucontext_t!  I don't know
+// how we'd get the PC (using StackWalk64?)
+//    http://msdn2.microsoft.com/en-us/library/ms680650.aspx
+
+#include "base/logging.h"   // for RAW_LOG
+#ifndef HAVE_CYGWIN_SIGNAL_H
+typedef int ucontext_t;
+#endif
+
+inline void* GetPC(ucontext_t *signal_ucontext) {
+  RAW_LOG(ERROR, "GetPC is not yet implemented on Windows\n");
+  return NULL;
+}
+
+// Normal cases.  If this doesn't compile, it's probably because
+// PC_FROM_UCONTEXT is the empty string.  You need to figure out
+// the right value for your system, and add it to the list in
+// configure.ac (or set it manually in your config.h).
+#else
+inline void* GetPC(ucontext_t *signal_ucontext) {
+  return (void*)signal_ucontext->PC_FROM_UCONTEXT;   // defined in config.h
+}
+
+#endif
+
+#endif  // BASE_GETPC_H_
diff --git a/pypy/module/_vmprof/src/trampoline.asmgcc.s b/pypy/module/_vmprof/src/trampoline.asmgcc.s
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/src/trampoline.asmgcc.s
@@ -0,0 +1,17 @@
+// NOTE: you need to use TABs, not spaces!
+        
+	.text
+	.p2align 4,,-1
+	.globl	pypy_execute_frame_trampoline
+	.type	pypy_execute_frame_trampoline, @function
+pypy_execute_frame_trampoline:
+	.cfi_startproc
+	pushq	%rdi
+	.cfi_def_cfa_offset 16
+	call pypy_pyframe_execute_frame at PLT
+	/* GCROOT 0(%rsp) */
+	popq	%rdi
+	.cfi_def_cfa_offset 8
+	ret
+	.cfi_endproc
+	.size	pypy_execute_frame_trampoline, .-pypy_execute_frame_trampoline
diff --git a/pypy/module/_vmprof/src/trampoline.h b/pypy/module/_vmprof/src/trampoline.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/src/trampoline.h
@@ -0,0 +1,1 @@
+void* pypy_execute_frame_trampoline(void*, void*, void*);
diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/src/vmprof.c
@@ -0,0 +1,340 @@
+/* VMPROF
+ *
+ * statistical sampling profiler specifically designed to profile programs
+ * which run on a Virtual Machine and/or bytecode interpreter, such as Python,
+ * etc.
+ *
+ * The logic to dump the C stack traces is partly stolen from the code in gperftools.
+ * The file "getpc.h" has been entirely copied from gperftools.
+ *
+ * Tested only on gcc, linux, x86_64.
+ *
+ * Copyright (C) 2014-2015
+ *   Antonio Cuni - anto.cuni at gmail.com
+ *   Maciej Fijalkowski - fijall at gmail.com
+ *
+ */
+
+
+#include "getpc.h"      // should be first to get the _GNU_SOURCE dfn
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#define UNW_LOCAL_ONLY
+#include 
+
+#include "vmprof.h"
+
+#define _unused(x) ((void)x)
+
+#define MAX_FUNC_NAME 128
+#define MAX_STACK_DEPTH 64
+
+static FILE* profile_file = NULL;
+void* vmprof_mainloop_func;
+static ptrdiff_t mainloop_sp_offset;
+static vmprof_get_virtual_ip_t mainloop_get_virtual_ip;
+
+
+/* *************************************************************
+ * functions to write a profile file compatible with gperftools
+ * *************************************************************
+ */
+
+#define MARKER_STACKTRACE '\x01'
+#define MARKER_VIRTUAL_IP '\x02'
+#define MARKER_TRAILER '\x03'
+
+static void prof_word(FILE* f, long x) {
+    fwrite(&x, sizeof(x), 1, f);
+}
+
+static void prof_header(FILE* f, long period_usec) {
+    prof_word(f, 0);
+    prof_word(f, 3);
+    prof_word(f, 0);
+    prof_word(f, period_usec);
+    prof_word(f, 0);
+}
+
+static void prof_write_stacktrace(FILE* f, void** stack, int depth, int count) {
+    int i;
+	char marker = MARKER_STACKTRACE;
+
+	fwrite(&marker, 1, 1, f);
+    prof_word(f, count);
+    prof_word(f, depth);
+    for(i=0; isp = bp;
+		bp -= sizeof(void*);
+        cp2->ip = ((void**)bp)[0];
+        // the ret is on the top of the stack minus WORD
+        return 1;
+    }
+}
+
+
+/* *************************************************************
+ * functions to dump the stack trace
+ * *************************************************************
+ */
+
+// stolen from pprof:
+// Sometimes, we can try to get a stack trace from within a stack
+// trace, because libunwind can call mmap (maybe indirectly via an
+// internal mmap based memory allocator), and that mmap gets trapped
+// and causes a stack-trace request.  If were to try to honor that
+// recursive request, we'd end up with infinite recursion or deadlock.
+// Luckily, it's safe to ignore those subsequent traces.  In such
+// cases, we return 0 to indicate the situation.
+//static __thread int recursive;
+static int recursive; // XXX antocuni: removed __thread
+
+int get_stack_trace(void** result, int max_depth, ucontext_t *ucontext) {
+    void *ip;
+    int n = 0;
+    unw_cursor_t cursor;
+    unw_context_t uc = *ucontext;
+    if (recursive) {
+        return 0;
+    }
+    ++recursive;
+
+    int ret = unw_init_local(&cursor, &uc);
+    assert(ret >= 0);
+    _unused(ret);
+
+    while (n < max_depth) {
+        if (unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *) &ip) < 0) {
+            break;
+        }
+
+        unw_proc_info_t pip;
+        unw_get_proc_info(&cursor, &pip);
+
+        /* char funcname[4096]; */
+        /* unw_word_t offset; */
+        /* unw_get_proc_name(&cursor, funcname, 4096, &offset); */
+        /* printf("%s+%#lx <%p>\n", funcname, offset, ip); */
+
+        /* if n==0, it means that the signal handler interrupted us while we
+           were in the trampoline, so we are not executing (yet) the real main
+           loop function; just skip it */
+        if (vmprof_mainloop_func && 
+            (void*)pip.start_ip == (void*)vmprof_mainloop_func &&
+            n > 0) {
+          // found main loop stack frame
+          void* sp;
+          unw_get_reg(&cursor, UNW_REG_SP, (unw_word_t *) &sp);
+          void *arg_addr = (char*)sp + mainloop_sp_offset;
+          void **arg_ptr = (void**)arg_addr;
+          // fprintf(stderr, "stacktrace mainloop: rsp %p   &f2 %p   offset %ld\n", 
+          //         sp, arg_addr, mainloop_sp_offset);
+          ip = mainloop_get_virtual_ip(*arg_ptr);
+        }
+
+        result[n++] = ip;
+		n = vmprof_write_header_for_jit_addr(result, n, ip, max_depth);
+        if (vmprof_unw_step(&cursor) <= 0) {
+            break;
+        }
+    }
+    --recursive;
+    return n;
+}
+
+
+static int __attribute__((noinline)) frame_forcer(int rv) {
+    return rv;
+}
+
+static void sigprof_handler(int sig_nr, siginfo_t* info, void *ucontext) {
+    void* stack[MAX_STACK_DEPTH];
+    stack[0] = GetPC((ucontext_t*)ucontext);
+    int depth = frame_forcer(get_stack_trace(stack+1, MAX_STACK_DEPTH-1, ucontext));
+    depth++;  // To account for pc value in stack[0];
+    prof_write_stacktrace(profile_file, stack, depth, 1);
+}
+
+/* *************************************************************
+ * functions to enable/disable the profiler
+ * *************************************************************
+ */
+
+static int open_profile(int fd, long period_usec, int write_header, char *s,
+						int slen) {
+	if ((fd = dup(fd)) == -1) {
+		return -1;
+	}
+    profile_file = fdopen(fd, "wb");
+	if (!profile_file) {
+		return -1;
+	}
+	if (write_header)
+		prof_header(profile_file, period_usec);
+	if (s)
+		fwrite(s, slen, 1, profile_file);
+	return 0;
+}
+
+static int close_profile(void) {
+	// XXX all of this can happily fail
+    FILE* src;
+    char buf[BUFSIZ];
+    size_t size;
+	int marker = MARKER_TRAILER;
+	fwrite(&marker, 1, 1, profile_file);
+
+    // copy /proc/PID/maps to the end of the profile file
+    sprintf(buf, "/proc/%d/maps", getpid());
+    src = fopen(buf, "r");    
+    while ((size = fread(buf, 1, BUFSIZ, src))) {
+        fwrite(buf, 1, size, profile_file);
+    }
+    fclose(src);
+    fclose(profile_file);
+	return 0;
+}
+
+
+static int install_sigprof_handler(void) {
+    struct sigaction sa;
+    memset(&sa, 0, sizeof(sa));
+    sa.sa_sigaction = sigprof_handler;
+    sa.sa_flags = SA_RESTART | SA_SIGINFO;
+    if (sigemptyset(&sa.sa_mask) == -1 ||
+		sigaction(SIGPROF, &sa, NULL) == -1) {
+		return -1;
+	}
+	return 0;
+}
+
+static int remove_sigprof_handler(void) {
+    //sighandler_t res = signal(SIGPROF, SIG_DFL);
+	//if (res == SIG_ERR) {
+	//	return -1;
+	//}
+	return 0;
+};
+
+static int install_sigprof_timer(long period_usec) {
+    static struct itimerval timer;
+    timer.it_interval.tv_sec = 0;
+    timer.it_interval.tv_usec = period_usec;
+    timer.it_value = timer.it_interval;
+    if (setitimer(ITIMER_PROF, &timer, NULL) != 0) {
+		return -1;
+    }
+	return 0;
+}
+
+static int remove_sigprof_timer(void) {
+    static struct itimerval timer;
+    timer.it_interval.tv_sec = 0;
+    timer.it_interval.tv_usec = 0;
+    timer.it_value = timer.it_interval;
+    if (setitimer(ITIMER_PROF, &timer, NULL) != 0) {
+		return -1;
+    }
+	return 0;
+}
+
+/* *************************************************************
+ * public API
+ * *************************************************************
+ */
+
+void vmprof_set_mainloop(void* func, ptrdiff_t sp_offset, 
+                         vmprof_get_virtual_ip_t get_virtual_ip) {
+    mainloop_sp_offset = sp_offset;
+    mainloop_get_virtual_ip = get_virtual_ip;
+    vmprof_mainloop_func = func;
+}
+
+int vmprof_enable(int fd, long period_usec, int write_header, char *s,
+				  int slen)
+{
+    if (period_usec == -1)
+        period_usec = 1000000 / 100; /* 100hz */
+    if (open_profile(fd, period_usec, write_header, s, slen) == -1) {
+		return -1;
+	}
+    if (install_sigprof_handler() == -1) {
+		return -1;
+	}
+    if (install_sigprof_timer(period_usec) == -1) {
+		return -1;
+	}
+	return 0;
+}
+
+int vmprof_disable(void) {
+    if (remove_sigprof_timer() == -1) {
+		return -1;
+	}
+    if (remove_sigprof_handler() == -1) {
+		return -1;
+	}
+    if (close_profile() == -1) {
+		return -1;
+	}
+	return 0;
+}
+
+void vmprof_register_virtual_function(const char* name, void* start, void* end) {
+    // for now *end is simply ignored
+	char buf[1024];
+	int lgt = strlen(name) + 2 * sizeof(long) + 1;
+
+	if (lgt > 1024) {
+		lgt = 1024;
+	}
+	buf[0] = MARKER_VIRTUAL_IP;
+	((void **)(((void*)buf) + 1))[0] = start;
+	((long *)(((void*)buf) + 1 + sizeof(long)))[0] = lgt - 2 * sizeof(long) - 1;
+	strncpy(buf + 2 * sizeof(long) + 1, name, 1024 - 2 * sizeof(long) - 1);
+	fwrite(buf, lgt, 1, profile_file);
+}
diff --git a/pypy/module/_vmprof/src/vmprof.h b/pypy/module/_vmprof/src/vmprof.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/src/vmprof.h
@@ -0,0 +1,22 @@
+#ifndef VMPROF_VMPROF_H_
+#define VMPROF_VMPROF_H_
+
+#include 
+
+typedef void* (*vmprof_get_virtual_ip_t)(void*);
+
+extern void* vmprof_mainloop_func;
+void vmprof_set_mainloop(void* func, ptrdiff_t sp_offset, 
+                         vmprof_get_virtual_ip_t get_virtual_ip);
+
+void vmprof_register_virtual_function(const char* name, void* start, void* end);
+
+
+int vmprof_enable(int fd, long period_usec, int write_header, char* vips,
+				  int vips_len);
+int vmprof_disable(void);
+
+// XXX: this should be part of _vmprof (the CPython extension), not vmprof (the library)
+void vmprof_set_tramp_range(void* start, void* end);
+
+#endif
diff --git a/pypy/module/_vmprof/test/__init__.py b/pypy/module/_vmprof/test/__init__.py
new file mode 100644
diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/test/test__vmprof.py
@@ -0,0 +1,55 @@
+
+import tempfile
+from pypy.tool.pytest.objspace import gettestobjspace
+
+class AppTestVMProf(object):
+    def setup_class(cls):
+        cls.space = gettestobjspace(usemodules=['_vmprof', 'struct'])
+        cls.tmpfile = tempfile.NamedTemporaryFile()
+        cls.w_tmpfileno = cls.space.wrap(cls.tmpfile.fileno())
+        cls.w_tmpfilename = cls.space.wrap(cls.tmpfile.name)
+        cls.tmpfile2 = tempfile.NamedTemporaryFile()
+        cls.w_tmpfileno2 = cls.space.wrap(cls.tmpfile2.fileno())
+        cls.w_tmpfilename2 = cls.space.wrap(cls.tmpfile2.name)
+
+    def test_import_vmprof(self):
+        import struct, sys
+
+        WORD = struct.calcsize('l')
+        
+        def count(s):
+            i = 0
+            count = 0
+            i += 5 * WORD # header
+            while i < len(s):
+                assert s[i] == '\x02'
+                i += 1
+                _, size = struct.unpack("ll", s[i:i + 2 * WORD])
+                count += 1
+                i += 2 * WORD + size
+            return count
+        
+        import _vmprof
+        _vmprof.enable(self.tmpfileno)
+        _vmprof.disable()
+        s = open(self.tmpfilename).read()
+        no_of_codes = count(s)
+        assert no_of_codes > 10
+        d = {}
+
+        exec """def foo():
+            pass
+        """ in d
+
+        _vmprof.enable(self.tmpfileno2)
+
+        exec """def foo2():
+            pass
+        """ in d
+
+        _vmprof.disable()
+        s = open(self.tmpfilename2).read()
+        no_of_codes2 = count(s)
+        assert "py:foo:" in s
+        assert "py:foo2:" in s
+        assert no_of_codes2 >= no_of_codes + 2 # some extra codes from tests
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -35,6 +35,9 @@
     name = opcode_method_names[ord(bytecode.co_code[next_instr])]
     return '%s #%d %s' % (bytecode.get_repr(), next_instr, name)
 
+def get_unique_id(next_instr, is_being_profiled, bytecode):
+    return bytecode._unique_id
+
 
 def should_unroll_one_iteration(next_instr, is_being_profiled, bytecode):
     return (bytecode.co_flags & CO_GENERATOR) != 0
@@ -45,6 +48,7 @@
     virtualizables = ['frame']
 
 pypyjitdriver = PyPyJitDriver(get_printable_location = get_printable_location,
+                              get_unique_id = get_unique_id,
                               should_unroll_one_iteration =
                               should_unroll_one_iteration,
                               name='pypyjit')
diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -57,6 +57,7 @@
         BaseAssembler.setup_once(self)
 
     def setup(self, looptoken):
+        BaseAssembler.setup(self, looptoken)
         assert self.memcpy_addr != 0, 'setup_once() not called?'
         if we_are_translated():
             self.debug = False
@@ -71,7 +72,6 @@
         self.mc.datablockwrapper = self.datablockwrapper
         self.target_tokens_currently_compiling = {}
         self.frame_depth_to_patch = []
-        self._finish_gcmap = lltype.nullptr(jitframe.GCMAP)
 
     def teardown(self):
         self.current_clt = None
@@ -659,6 +659,7 @@
             assert len(set(inputargs)) == len(inputargs)
 
         self.setup(original_loop_token)
+        self.codemap.inherit_code_from_position(faildescr.adr_jump_offset)
         descr_number = compute_unique_id(faildescr)
         if log:
             operations = self._inject_debugging_code(faildescr, operations,
@@ -879,8 +880,12 @@
         self.datablockwrapper.done()      # finish using cpu.asmmemmgr
         self.datablockwrapper = None
         allblocks = self.get_asmmemmgr_blocks(looptoken)
-        return self.mc.materialize(self.cpu.asmmemmgr, allblocks,
+        size = self.mc.get_relative_pos() 
+        res = self.mc.materialize(self.cpu.asmmemmgr, allblocks,
                                    self.cpu.gc_ll_descr.gcrootmap)
+        self.cpu.asmmemmgr.register_codemap(
+            self.codemap.get_final_bytecode(res, size))
+        return res
 
     def update_frame_depth(self, frame_depth):
         baseofs = self.cpu.get_baseofs_of_frame_field()
diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -392,6 +392,9 @@
         else:
             self.rm._sync_var(v)
 
+    def prepare_op_debug_merge_point(self, op, fcond):
+        self.assembler.codemap.debug_merge_point(op)
+
     def _prepare_op_int_add(self, op, fcond):
         boxes = op.getarglist()
         a0, a1 = boxes
diff --git a/rpython/jit/backend/llsupport/asmmemmgr.py b/rpython/jit/backend/llsupport/asmmemmgr.py
--- a/rpython/jit/backend/llsupport/asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/asmmemmgr.py
@@ -4,7 +4,10 @@
 from rpython.rlib import rmmap
 from rpython.rlib.debug import debug_start, debug_print, debug_stop
 from rpython.rlib.debug import have_debug_prints
-from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rlib.rbisect import bisect, bisect_tuple
+
+_memmngr = None # global reference so we can use @entrypoint :/
 
 
 class AsmMemoryManager(object):
@@ -24,6 +27,12 @@
         self.free_blocks = {}      # map {start: stop}
         self.free_blocks_end = {}  # map {stop: start}
         self.blocks_by_size = [[] for i in range(self.num_indices)]
+        # two lists of jit addresses (sorted) and the corresponding stack
+        # depths
+        self.jit_addr_map = []
+        self.jit_frame_depth_map = []
+        self.jit_codemap = []
+        # see codemap.py
 
     def malloc(self, minsize, maxsize):
         """Allocate executable memory, between minsize and maxsize bytes,
@@ -45,6 +54,13 @@
         if r_uint is not None:
             self.total_mallocs -= r_uint(stop - start)
         self._add_free_block(start, stop)
+        # fix up jit_addr_map
+        jit_adr_start = bisect(self.jit_addr_map, start)
+        jit_adr_stop = bisect(self.jit_addr_map, stop)
+        self.jit_addr_map = (self.jit_addr_map[:jit_adr_start] +
+                             self.jit_addr_map[jit_adr_stop:])
+        self.jit_frame_depth_map = (self.jit_frame_depth_map[:jit_adr_start] +
+                                    self.jit_frame_depth_map[jit_adr_stop:])
 
     def open_malloc(self, minsize):
         """Allocate at least minsize bytes.  Returns (start, stop)."""
@@ -151,6 +167,35 @@
         del self.free_blocks_end[stop]
         return (start, stop)
 
+    def register_frame_depth_map(self, rawstart, frame_positions,
+                                 frame_assignments):
+        if not frame_positions:
+            return
+        if not self.jit_addr_map or rawstart > self.jit_addr_map[-1]:
+            start = len(self.jit_addr_map)
+            self.jit_addr_map += [0] * len(frame_positions)
+            self.jit_frame_depth_map += [0] * len(frame_positions)
+        else:
+            start = bisect(self.jit_addr_map, rawstart)
+            self.jit_addr_map = (self.jit_addr_map[:start] +
+                                 [0] * len(frame_positions) +
+                                 self.jit_addr_map[start:])
+            self.jit_frame_depth_map = (self.jit_frame_depth_map[:start] +
+                                 [0] * len(frame_positions) +
+                                 self.jit_frame_depth_map[start:])
+        for i, pos in enumerate(frame_positions):
+            self.jit_addr_map[i + start] = pos + rawstart
+            self.jit_frame_depth_map[i + start] = frame_assignments[i]
+
+    def register_codemap(self, codemap):
+        start = codemap[0]
+        pos = bisect_tuple(self.jit_codemap, start)
+        if pos == len(self.jit_codemap): # common case
+            self.jit_codemap.append(codemap)
+        else:
+            self.jit_codemap = (self.jit_codemap[:pos] + [codemap] +
+                                self.jit_codemap[pos:])
+
     def _delete(self):
         "NOT_RPYTHON"
         if self._allocated:
@@ -212,6 +257,9 @@
 
     gcroot_markers = None
 
+    frame_positions = None
+    frame_assignments = None
+
     def __init__(self, translated=None):
         if translated is None:
             translated = we_are_translated()
@@ -316,6 +364,10 @@
             assert gcrootmap is not None
             for pos, mark in self.gcroot_markers:
                 gcrootmap.register_asm_addr(rawstart + pos, mark)
+        asmmemmgr.register_frame_depth_map(rawstart, self.frame_positions,
+                                           self.frame_assignments)
+        self.frame_positions = None
+        self.frame_assignments = None
         return rawstart
 
     def _become_a_plain_block_builder(self):
diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -1,6 +1,7 @@
 from rpython.jit.backend.llsupport import jitframe
 from rpython.jit.backend.llsupport.memcpy import memcpy_fn, memset_fn
 from rpython.jit.backend.llsupport.symbolic import WORD
+from rpython.jit.backend.llsupport.codemap import CodemapBuilder
 from rpython.jit.metainterp.history import (INT, REF, FLOAT, JitCellToken,
     ConstInt, BoxInt, AbstractFailDescr)
 from rpython.jit.metainterp.resoperation import ResOperation, rop
@@ -128,6 +129,10 @@
                                               track_allocation=False)
         self.gcmap_for_finish[0] = r_uint(1)
 
+    def setup(self, looptoken):
+        self.codemap = CodemapBuilder()
+        self._finish_gcmap = lltype.nullptr(jitframe.GCMAP)
+
     def set_debug(self, v):
         r = self._debug
         self._debug = v
@@ -194,6 +199,9 @@
         guardtok.faildescr.rd_locs = positions
         return fail_descr, target
 
+    def debug_merge_point(self, op):
+        self.codemap.debug_merge_point(op, self.mc.get_relative_pos())
+
     def call_assembler(self, op, guard_op, argloc, vloc, result_loc, tmploc):
         self._store_force_index(guard_op)
         descr = op.getdescr()
@@ -276,6 +284,9 @@
         # YYY very minor leak -- we need the counters to stay alive
         # forever, just because we want to report them at the end
         # of the process
+
+        # XXX the numbers here are ALMOST unique, but not quite, use a counter
+        #     or something
         struct = lltype.malloc(DEBUG_COUNTER, flavor='raw',
                                track_allocation=False)
         struct.i = 0
diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/llsupport/codemap.py
@@ -0,0 +1,142 @@
+
+""" Bytecode for storage in asmmemmgr.jit_codemap. Format is as follows:
+
+ list of tuples of shape (addr, machine code size, bytecode info)
+ where bytecode info is a string made up of:
+    8 bytes unique_id, 4 bytes start_addr (relative), 4 bytes size (relative),
+    2 bytes how many items to skip to go to the next on similar level
+    [so far represented by a list of integers for simplicity]
+
+"""
+
+from rpython.rlib import rgc
+from rpython.rlib.entrypoint import jit_entrypoint
+from rpython.jit.backend.llsupport import asmmemmgr
+from rpython.rlib.rbisect import bisect, bisect_tuple
+from rpython.rtyper.lltypesystem import lltype, rffi
+
+ at jit_entrypoint([lltype.Signed], lltype.Signed,
+                c_name='pypy_jit_stack_depth_at_loc')
+ at rgc.no_collect
+def stack_depth_at_loc(loc):
+    _memmngr = asmmemmgr._memmngr
+
+    pos = bisect(_memmngr.jit_addr_map, loc)
+    if pos == 0 or pos == len(_memmngr.jit_addr_map):
+        return -1
+    return _memmngr.jit_frame_depth_map[pos-1]
+
+ at jit_entrypoint([], lltype.Signed, c_name='pypy_jit_start_addr')
+def jit_start_addr():
+    _memmngr = asmmemmgr._memmngr
+
+    return _memmngr.jit_addr_map[0]
+
+ at jit_entrypoint([], lltype.Signed, c_name='pypy_jit_end_addr')
+def jit_end_addr():
+    _memmngr = asmmemmgr._memmngr
+
+    return _memmngr.jit_addr_map[-1]
+
+ at jit_entrypoint([lltype.Signed], lltype.Signed,
+                c_name='pypy_find_codemap_at_addr')
+def find_codemap_at_addr(addr):
+    _memmngr = asmmemmgr._memmngr
+
+    res = bisect_tuple(_memmngr.jit_codemap, addr) - 1
+    if res == len(_memmngr.jit_codemap):
+        return -1
+    return res
+
+ at jit_entrypoint([lltype.Signed, lltype.Signed,
+                 rffi.CArrayPtr(lltype.Signed)], lltype.Signed,
+                 c_name='pypy_yield_codemap_at_addr')
+def yield_bytecode_at_addr(codemap_no, addr, current_pos_addr):
+    """ will return consecutive unique_ids from codemap, starting from position
+    `pos` until addr
+    """
+    _memmngr = asmmemmgr._memmngr
+
+    codemap = _memmngr.jit_codemap[codemap_no]
+    current_pos = current_pos_addr[0]
+    start_addr = codemap[0]
+    rel_addr = addr - start_addr
+    while True:
+        if current_pos >= len(codemap[2]):
+            return 0
+        next_start = codemap[2][current_pos + 1]
+        if next_start > rel_addr:
+            return 0
+        next_stop = codemap[2][current_pos + 2]
+        if next_stop > rel_addr:
+            current_pos_addr[0] = current_pos + 4
+            return codemap[2][current_pos]
+        # we need to skip potentially more than one
+        current_pos = codemap[2][current_pos + 3]
+
+def unpack_traceback(addr):
+    codemap_pos = find_codemap_at_addr(addr)
+    assert codemap_pos >= 0
+    storage = lltype.malloc(rffi.CArray(lltype.Signed), 1, flavor='raw')
+    storage[0] = 0
+    res = []
+    while True:
+        item = yield_bytecode_at_addr(codemap_pos, addr, storage)
+        if item == 0:
+            break
+        res.append(item)
+    lltype.free(storage, flavor='raw')
+    return res
+
+
+class CodemapBuilder(object):
+    def __init__(self):
+        self.l = []
+        self.patch_position = []
+        self.last_call_depth = -1
+
+    def debug_merge_point(self, op, pos):
+        call_depth = op.getarg(1).getint()
+        if call_depth != self.last_call_depth:
+            unique_id = op.getarg(3).getint()
+            if unique_id == 0: # uninteresting case
+                return
+            assert unique_id & 1 == 0
+            if call_depth > self.last_call_depth:
+                self.l.append(unique_id)
+                self.l.append(pos) # <- this is a relative pos
+                self.patch_position.append(len(self.l))
+                self.l.append(0) # marker
+                self.l.append(0) # second marker
+            else:
+                for i in range(self.last_call_depth - call_depth):
+                    to_patch = self.patch_position.pop()
+                    self.l[to_patch] = pos
+                    self.l[to_patch + 1] = len(self.l)
+            self.last_call_depth = call_depth
+
+    def inherit_code_from_position(self, pos):
+        lst = unpack_traceback(pos)
+        self.last_call_depth = len(lst) - 1
+        for item in lst:
+            self.l.append(item)
+            self.l.append(0)
+            self.patch_position.append(len(self.l))
+            self.l.append(0) # marker
+            self.l.append(0) # second marker
+
+    def get_final_bytecode(self, addr, size):
+        while self.patch_position:
+            pos = self.patch_position.pop()
+            self.l[pos] = size
+            self.l[pos + 1] = len(self.l)
+        # at the end there should be no zeros
+        for i in range(len(self.l) / 4):
+            item = self.l[i * 4] # unique_id
+            assert item > 0 # no zeros here
+            item = self.l[i * 4 + 2] # end in asm
+            assert item > 0
+            item = self.l[i * 4 + 3] # end in l
+            assert item > 0
+        return (addr, size, self.l) # XXX compact self.l
+
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -16,7 +16,7 @@
     FieldDescr, ArrayDescr, CallDescr, InteriorFieldDescr,
     FLAG_POINTER, FLAG_FLOAT)
 from rpython.jit.backend.llsupport.memcpy import memset_fn
-from rpython.jit.backend.llsupport.asmmemmgr import AsmMemoryManager
+from rpython.jit.backend.llsupport import asmmemmgr
 from rpython.rlib.unroll import unrolling_iterable
 
 
@@ -48,7 +48,8 @@
             self._setup_exception_handling_translated()
         else:
             self._setup_exception_handling_untranslated()
-        self.asmmemmgr = AsmMemoryManager()
+        self.asmmemmgr = asmmemmgr.AsmMemoryManager()
+        asmmemmgr._memmngr = self.asmmemmgr
         self._setup_frame_realloc(translate_support_code)
         ad = self.gc_ll_descr.getframedescrs(self).arraydescr
         self.signedarraydescr = ad
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -17,8 +17,6 @@
 class GcRewriterAssembler(object):
     """ This class performs the following rewrites on the list of operations:
 
-     - Remove the DEBUG_MERGE_POINTs.
-
      - Turn all NEW_xxx to either a CALL_MALLOC_GC, or a CALL_MALLOC_NURSERY
        followed by SETFIELDs in order to initialize their GC fields.  The
        two advantages of CALL_MALLOC_NURSERY is that it inlines the common
@@ -60,8 +58,6 @@
         #
         for i in range(len(operations)):
             op = operations[i]
-            if op.getopnum() == rop.DEBUG_MERGE_POINT:
-                continue
             # ---------- turn NEWxxx into CALL_MALLOC_xxx ----------
             if op.is_malloc():
                 self.handle_malloc_operation(op)
diff --git a/rpython/jit/backend/llsupport/test/test_asmmemmgr.py b/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
--- a/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
@@ -2,6 +2,7 @@
 from rpython.jit.backend.llsupport.asmmemmgr import AsmMemoryManager
 from rpython.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
 from rpython.jit.backend.llsupport.asmmemmgr import BlockBuilderMixin
+from rpython.jit.backend.llsupport import asmmemmgr
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib import debug
 
@@ -157,6 +158,7 @@
         class FakeGcRootMap:
             def register_asm_addr(self, retaddr, mark):
                 puts.append((retaddr, mark))
+
         #
         mc = BlockBuilderMixin()
         mc.writechar('X')
@@ -266,3 +268,16 @@
     md.done()
     assert allblocks == [(1597, 1697), (1797, 1835)]
     assert ops == [('free', 1835, 1897)]
+
+def test_find_jit_frame_depth():
+    mgr = AsmMemoryManager()
+    mgr.register_frame_depth_map(11, [0, 5, 10], [1, 2, 3])
+    mgr.register_frame_depth_map(30, [0, 5, 10], [4, 5, 6])
+    mgr.register_frame_depth_map(0, [0, 5, 10], [7, 8, 9])
+    asmmemmgr._memmngr = mgr
+    assert asmmemmgr.stack_depth_at_loc(13) == 1
+    assert asmmemmgr.stack_depth_at_loc(-3) == -1
+    assert asmmemmgr.stack_depth_at_loc(41) == -1
+    assert asmmemmgr.stack_depth_at_loc(5) == 8
+    assert asmmemmgr.stack_depth_at_loc(17) == 2
+    assert asmmemmgr.stack_depth_at_loc(38) == 5
diff --git a/rpython/jit/backend/llsupport/test/ztranslation_test.py b/rpython/jit/backend/llsupport/test/ztranslation_test.py
--- a/rpython/jit/backend/llsupport/test/ztranslation_test.py
+++ b/rpython/jit/backend/llsupport/test/ztranslation_test.py
@@ -2,7 +2,7 @@
 from rpython.tool.udir import udir
 from rpython.rlib.jit import JitDriver, unroll_parameters, set_param
 from rpython.rlib.jit import PARAMETERS, dont_look_inside
-from rpython.rlib.jit import promote
+from rpython.rlib.jit import promote, _get_virtualizable_token
 from rpython.rlib import jit_hooks, rposix
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib.rthread import ThreadLocalReference, ThreadLocalField
@@ -28,12 +28,15 @@
         # - floats neg and abs
         # - llexternal with macro=True
 
-        class Frame(object):
+        class BasicFrame(object):
             _virtualizable_ = ['i']
 
             def __init__(self, i):
                 self.i = i
 
+        class Frame(BasicFrame):
+            pass
+
         eci = ExternalCompilationInfo(post_include_bits=['''
 #define pypy_my_fabs(x)  fabs(x)
 '''])
@@ -59,6 +62,7 @@
             while frame.i > 3:
                 jitdriver.can_enter_jit(frame=frame, total=total, j=j)
                 jitdriver.jit_merge_point(frame=frame, total=total, j=j)
+                _get_virtualizable_token(frame)
                 total += frame.i
                 if frame.i >= 20:
                     frame.i -= 2
diff --git a/rpython/jit/backend/x86/arch.py b/rpython/jit/backend/x86/arch.py
--- a/rpython/jit/backend/x86/arch.py
+++ b/rpython/jit/backend/x86/arch.py
@@ -51,3 +51,6 @@
     THREADLOCAL_OFS = (FRAME_FIXED_SIZE - 1) * WORD
 
 assert PASS_ON_MY_FRAME >= 12       # asmgcc needs at least JIT_USE_WORDS + 3
+
+# return address, followed by FRAME_FIXED_SIZE words
+DEFAULT_FRAME_BYTES = (1 + FRAME_FIXED_SIZE) * WORD
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -18,7 +18,8 @@
 from rpython.jit.backend.llsupport.regalloc import (get_scale, valid_addressing_size)
 from rpython.jit.backend.x86.arch import (FRAME_FIXED_SIZE, WORD, IS_X86_64,
                                        JITFRAME_FIXED_SIZE, IS_X86_32,
-                                       PASS_ON_MY_FRAME, THREADLOCAL_OFS)
+                                       PASS_ON_MY_FRAME, THREADLOCAL_OFS,
+                                       DEFAULT_FRAME_BYTES)
 from rpython.jit.backend.x86.regloc import (eax, ecx, edx, ebx, esp, ebp, esi,
     xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, r8, r9, r10, r11, edi,
     r12, r13, r14, r15, X86_64_SCRATCH_REG, X86_64_XMM_SCRATCH_REG,
@@ -66,6 +67,7 @@
             self._build_float_constants()
 
     def setup(self, looptoken):
+        BaseAssembler.setup(self, looptoken)
         assert self.memcpy_addr != 0, "setup_once() not called?"
         self.current_clt = looptoken.compiled_loop_token
         self.pending_guard_tokens = []
@@ -80,7 +82,6 @@
                                                         allblocks)
         self.target_tokens_currently_compiling = {}
         self.frame_depth_to_patch = []
-        self._finish_gcmap = lltype.nullptr(jitframe.GCMAP)
 
     def teardown(self):
         self.pending_guard_tokens = None
@@ -267,6 +268,10 @@
         # the correct "ret" arg
         offset = mc.get_relative_pos() - jz_location
         mc.overwrite32(jz_location-4, offset)
+        # From now on this function is basically "merged" with
+        # its caller and so contains DEFAULT_FRAME_BYTES bytes
+        # plus my own return address, which we'll ignore next
+        mc.force_frame_size(DEFAULT_FRAME_BYTES + WORD)
         mc.ADD_ri(esp.value, WORD)
         mc.JMP(imm(self.propagate_exception_path))
         #
@@ -278,6 +283,7 @@
             return      # not supported (for tests, or non-translated)
         #
         self.mc = codebuf.MachineCodeBlockWrapper()
+        self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         #
         # read and reset the current exception
 
@@ -331,7 +337,10 @@
         offset = mc.get_relative_pos() - jnz_location
         assert 0 < offset <= 127
         mc.overwrite(jnz_location-1, chr(offset))
-        # adjust the esp to point back to the previous return
+        # From now on this function is basically "merged" with
+        # its caller and so contains DEFAULT_FRAME_BYTES bytes
+        # plus my own return address, which we'll ignore next
+        mc.force_frame_size(DEFAULT_FRAME_BYTES + WORD)
         mc.ADD_ri(esp.value, WORD)
         mc.JMP(imm(self.propagate_exception_path))
         #
@@ -409,6 +418,8 @@
                 mc.LEA_rs(esp.value, 2 * WORD)
             self._pop_all_regs_from_frame(mc, [], withfloats, callee_only=True)
             mc.RET16_i(WORD)
+            # Note that wb_slowpath[0..3] end with a RET16_i, which must be
+            # taken care of in the caller by stack_frame_size_delta(-WORD)
         else:
             if IS_X86_32:
                 mc.MOV_rs(edx.value, 4 * WORD)
@@ -514,6 +525,8 @@
             assert len(set(inputargs)) == len(inputargs)
 
         self.setup(original_loop_token)
+        self.codemap.inherit_code_from_position(faildescr.adr_jump_offset)
+        self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         descr_number = compute_unique_id(faildescr)
         if log:
             operations = self._inject_debugging_code(faildescr, operations,
@@ -674,8 +687,12 @@
         self.datablockwrapper.done()      # finish using cpu.asmmemmgr
         self.datablockwrapper = None
         allblocks = self.get_asmmemmgr_blocks(looptoken)
-        return self.mc.materialize(self.cpu.asmmemmgr, allblocks,
-                                   self.cpu.gc_ll_descr.gcrootmap)
+        size = self.mc.get_relative_pos()
+        res = self.mc.materialize(self.cpu.asmmemmgr, allblocks,
+                                  self.cpu.gc_ll_descr.gcrootmap)
+        self.cpu.asmmemmgr.register_codemap(
+            self.codemap.get_final_bytecode(res, size))
+        return res
 
     def patch_jump_for_descr(self, faildescr, adr_new_target):
         adr_jump_offset = faildescr.adr_jump_offset
@@ -686,6 +703,7 @@
         # place, but clobber the recovery stub with a jump to the real
         # target.
         mc = codebuf.MachineCodeBlockWrapper()
+        mc.force_frame_size(DEFAULT_FRAME_BYTES)
         if rx86.fits_in_32bits(offset):
             mc.writeimm32(offset)
             mc.copy_to_raw_memory(adr_jump_offset)
@@ -1756,6 +1774,7 @@
 
     def generate_propagate_error_64(self):
         assert WORD == 8
+        self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         startpos = self.mc.get_relative_pos()
         self.mc.JMP(imm(self.propagate_exception_path))
         return startpos
@@ -1763,6 +1782,7 @@
     def generate_quick_failure(self, guardtok):
         """ Gather information about failure
         """
+        self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         startpos = self.mc.get_relative_pos()
         fail_descr, target = self.store_info_on_descr(startpos, guardtok)
         self.mc.PUSH(imm(fail_descr))
@@ -1838,6 +1858,9 @@
 
     def _build_failure_recovery(self, exc, withfloats=False):
         mc = codebuf.MachineCodeBlockWrapper()
+        # this is jumped to, from a stack that has DEFAULT_FRAME_BYTES
+        # followed by 2 extra words just pushed
+        mc.force_frame_size(DEFAULT_FRAME_BYTES + 2 * WORD)
         self.mc = mc
 
         self._push_all_regs_to_frame(mc, [], withfloats)
@@ -1909,6 +1932,7 @@
             self.mc.J_il(rx86.Conditions[condition], 0)
         else:
             self.mc.JMP_l(0)
+            self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         guard_token.pos_jump_offset = self.mc.get_relative_pos() - 4
         self.pending_guard_tokens.append(guard_token)
 
@@ -2020,6 +2044,7 @@
         offset = jmp_location - je_location
         assert 0 < offset <= 127
         self.mc.overwrite(je_location - 1, chr(offset))
+        self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         #
         return jmp_location
 
@@ -2104,6 +2129,8 @@
         if is_frame and align_stack:
             mc.SUB_ri(esp.value, 16 - WORD) # erase the return address
         mc.CALL(imm(self.wb_slowpath[helper_num]))
+        if not is_frame:
+            mc.stack_frame_size_delta(-WORD)
         if is_frame and align_stack:
             mc.ADD_ri(esp.value, 16 - WORD) # erase the return address
 
@@ -2340,6 +2367,7 @@
         offset = self.mc.get_relative_pos() - jmp_adr1
         assert 0 < offset <= 127
         self.mc.overwrite(jmp_adr1-1, chr(offset))
+        self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         # write down the tid, but not if it's the result of the CALL
         self.mc.MOV(mem(eax, 0), imm(arraydescr.tid))
         # while we're at it, this line is not needed if we've done the CALL
diff --git a/rpython/jit/backend/x86/callbuilder.py b/rpython/jit/backend/x86/callbuilder.py
--- a/rpython/jit/backend/x86/callbuilder.py
+++ b/rpython/jit/backend/x86/callbuilder.py
@@ -58,7 +58,7 @@
             self.fnloc_is_immediate = False
             self.fnloc = None
             self.arglocs = arglocs + [fnloc]
-        self.current_esp = 0     # 0 or (usually) negative, counted in bytes
+        self.start_frame_size = self.mc._frame_size
 
     def select_call_release_gil_mode(self):
         AbstractCallBuilder.select_call_release_gil_mode(self)
@@ -70,13 +70,15 @@
     def subtract_esp_aligned(self, count):
         if count > 0:
             align = align_stack_words(count)
-            self.current_esp -= align * WORD
             self.mc.SUB_ri(esp.value, align * WORD)
 
+    def get_current_esp(self):
+        return self.start_frame_size - self.mc._frame_size
+
     def restore_stack_pointer(self, target_esp=0):
-        if self.current_esp != target_esp:
-            self.mc.ADD_ri(esp.value, target_esp - self.current_esp)
-            self.current_esp = target_esp
+        current_esp = self.get_current_esp()
+        if current_esp != target_esp:
+            self.mc.ADD_ri(esp.value, target_esp - current_esp)
 
     def load_result(self):
         """Overridden in CallBuilder32 and CallBuilder64"""
@@ -99,9 +101,10 @@
         # after the rearrangements done just before, ignoring the return
         # value eax, if necessary
         assert not self.is_call_release_gil
-        self.change_extra_stack_depth = (self.current_esp != 0)
+        current_esp = self.get_current_esp()
+        self.change_extra_stack_depth = (current_esp != 0)
         if self.change_extra_stack_depth:
-            self.asm.set_extra_stack_depth(self.mc, -self.current_esp)
+            self.asm.set_extra_stack_depth(self.mc, -current_esp)
         noregs = self.asm.cpu.gc_ll_descr.is_shadow_stack()
         gcmap = self.asm._regalloc.get_gcmap([eax], noregs=noregs)
         self.asm.push_gcmap(self.mc, gcmap, store=True)
@@ -142,7 +145,7 @@
             # and 5/7 words as described for asmgcroot.ASM_FRAMEDATA, for a
             # total size of JIT_USE_WORDS.  This structure is found at
             # [ESP+css].
-            css = -self.current_esp + (
+            css = -self.get_current_esp() + (
                 WORD * (PASS_ON_MY_FRAME - asmgcroot.JIT_USE_WORDS))
             assert css >= 2 * WORD
             # Save ebp
@@ -178,7 +181,7 @@
             else:
                 self.tlofs_reg = r12
             self.mc.MOV_rs(self.tlofs_reg.value,
-                           THREADLOCAL_OFS - self.current_esp)
+                           THREADLOCAL_OFS - self.get_current_esp())
             if self.asm._is_asmgcc():
                 self.mc.AND_ri(self.tlofs_reg.value, ~1)
         return self.tlofs_reg
@@ -451,7 +454,10 @@
         else:
             self.mc.CALL(self.fnloc)
             if self.callconv != FFI_DEFAULT_ABI:
-                self.current_esp += self._fix_stdcall(self.callconv)
+                # in the STDCALL ABI, the CALL above has an effect on
+                # the stack depth.  Adjust 'mc._frame_size'.
+                delta = self._fix_stdcall(self.callconv)
+                self.mc.stack_frame_size_delta(-delta)
 
     def _fix_stdcall(self, callconv):
         from rpython.rlib.clibffi import FFI_STDCALL
diff --git a/rpython/jit/backend/x86/codebuf.py b/rpython/jit/backend/x86/codebuf.py
--- a/rpython/jit/backend/x86/codebuf.py
+++ b/rpython/jit/backend/x86/codebuf.py
@@ -23,6 +23,7 @@
                               codebuilder_cls):
     def __init__(self):
         self.init_block_builder()
+        codebuilder_cls.__init__(self)
         # a list of relative positions; for each position p, the bytes
         # at [p-4:p] encode an absolute address that will need to be
         # made relative.  Only works on 32-bit!
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -12,7 +12,7 @@
      valid_addressing_size)
 from rpython.jit.backend.x86 import rx86
 from rpython.jit.backend.x86.arch import (WORD, JITFRAME_FIXED_SIZE, IS_X86_32,
-    IS_X86_64)
+    IS_X86_64, DEFAULT_FRAME_BYTES)
 from rpython.jit.backend.x86.jump import remap_frame_layout_mixed
 from rpython.jit.backend.x86.regloc import (FrameLoc, RegLoc, ConstFloatLoc,
     FloatImmedLoc, ImmedLoc, imm, imm0, imm1, ecx, eax, edx, ebx, esi, edi,
@@ -314,6 +314,7 @@
         while i < len(operations):
             op = operations[i]
             self.assembler.mc.mark_op(op)
+            assert self.assembler.mc._frame_size == DEFAULT_FRAME_BYTES
             self.rm.position = i
             self.xrm.position = i
             if op.has_no_side_effect() and op.result not in self.longevity:
@@ -1323,7 +1324,7 @@
         assembler.closing_jump(self.jump_target_descr)
 
     def consider_debug_merge_point(self, op):
-        pass
+        self.assembler.debug_merge_point(op)
 
     def consider_jit_debug(self, op):
         pass
diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py
--- a/rpython/jit/backend/x86/rx86.py
+++ b/rpython/jit/backend/x86/rx86.py
@@ -454,6 +454,11 @@
 class AbstractX86CodeBuilder(object):
     """Abstract base class."""
 
+    def __init__(self):
+        self.frame_positions = []
+        self.frame_assignments = []
+        self.force_frame_size(self.WORD)
+
     def writechar(self, char):
         raise NotImplementedError
 
@@ -471,6 +476,23 @@
         self.writechar(chr((imm >> 16) & 0xFF))
         self.writechar(chr((imm >> 24) & 0xFF))
 
+    def force_frame_size(self, frame_size):
+        self.frame_positions.append(self.get_relative_pos())
+        self.frame_assignments.append(frame_size)
+        self._frame_size = frame_size
+
+    def stack_frame_size_delta(self, delta):
+        "Called when we generate an instruction that changes the value of ESP"
+        self._frame_size += delta
+        self.frame_positions.append(self.get_relative_pos()) 
+        self.frame_assignments.append(self._frame_size)
+        assert self._frame_size >= self.WORD
+
+    def check_stack_size_at_ret(self):
+        assert self._frame_size == self.WORD
+        if not we_are_translated():
+            self._frame_size = None
+
     # ------------------------------ MOV ------------------------------
 
     MOV_ri = insn(register(1), '\xB8', immediate(2))
@@ -481,14 +503,24 @@
     INC_m = insn(rex_w, '\xFF', orbyte(0), mem_reg_plus_const(1))
     INC_j = insn(rex_w, '\xFF', orbyte(0), abs_(1))
 
-    ADD_ri,ADD_rr,ADD_rb,_,_,ADD_rm,ADD_rj,_,_ = common_modes(0)
+    AD1_ri,ADD_rr,ADD_rb,_,_,ADD_rm,ADD_rj,_,_ = common_modes(0)
     OR_ri, OR_rr, OR_rb, _,_,OR_rm, OR_rj, _,_ = common_modes(1)
     AND_ri,AND_rr,AND_rb,_,_,AND_rm,AND_rj,_,_ = common_modes(4)
-    SUB_ri,SUB_rr,SUB_rb,_,_,SUB_rm,SUB_rj,SUB_ji8,SUB_mi8 = common_modes(5)
+    SU1_ri,SUB_rr,SUB_rb,_,_,SUB_rm,SUB_rj,SUB_ji8,SUB_mi8 = common_modes(5)
     SBB_ri,SBB_rr,SBB_rb,_,_,SBB_rm,SBB_rj,_,_ = common_modes(3)
     XOR_ri,XOR_rr,XOR_rb,_,_,XOR_rm,XOR_rj,_,_ = common_modes(6)
     CMP_ri,CMP_rr,CMP_rb,CMP_bi,CMP_br,CMP_rm,CMP_rj,_,_ = common_modes(7)
 
+    def ADD_ri(self, reg, immed):
+        self.AD1_ri(reg, immed)
+        if reg == R.esp:
+            self.stack_frame_size_delta(-immed)
+
+    def SUB_ri(self, reg, immed):
+        self.SU1_ri(reg, immed)
+        if reg == R.esp:
+            self.stack_frame_size_delta(+immed)
+
     CMP_mi8 = insn(rex_w, '\x83', orbyte(7<<3), mem_reg_plus_const(1), immediate(2, 'b'))
     CMP_mi32 = insn(rex_w, '\x81', orbyte(7<<3), mem_reg_plus_const(1), immediate(2))
     CMP_mi = select_8_or_32_bit_immed(CMP_mi8, CMP_mi32)
@@ -538,30 +570,65 @@
     # ------------------------------ Misc stuff ------------------------------
 
     NOP = insn('\x90')
-    RET = insn('\xC3')
-    RET16_i = insn('\xC2', immediate(1, 'h'))
+    RE1 = insn('\xC3')
+    RE116_i = insn('\xC2', immediate(1, 'h'))
 
-    PUSH_r = insn(rex_nw, register(1), '\x50')
-    PUSH_b = insn(rex_nw, '\xFF', orbyte(6<<3), stack_bp(1))
-    PUSH_m = insn(rex_nw, '\xFF', orbyte(6<<3), mem_reg_plus_const(1))
-    PUSH_i8 = insn('\x6A', immediate(1, 'b'))
-    PUSH_i32 = insn('\x68', immediate(1, 'i'))
-    def PUSH_i(mc, immed):
+    def RET(self):
+        self.check_stack_size_at_ret()
+        self.RE1()
+
+    def RET16_i(self, immed):
+        self.check_stack_size_at_ret()
+        self.RE116_i(immed)
+
+    PUS1_r = insn(rex_nw, register(1), '\x50')
+    PUS1_b = insn(rex_nw, '\xFF', orbyte(6<<3), stack_bp(1))

From noreply at buildbot.pypy.org  Fri Apr 17 10:03:04 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 17 Apr 2015 10:03:04 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof2: reapply
Message-ID: <20150417080304.878641C1331@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof2
Changeset: r76814:1971d95c6746
Date: 2015-04-17 10:02 +0200
http://bitbucket.org/pypy/pypy/changeset/1971d95c6746/

Log:	reapply

diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -883,7 +883,7 @@
         size = self.mc.get_relative_pos() 
         res = self.mc.materialize(self.cpu.asmmemmgr, allblocks,
                                    self.cpu.gc_ll_descr.gcrootmap)
-        self.cpu.asmmemmgr.register_codemap(
+        self.cpu.codemap.register_codemap(
             self.codemap.get_final_bytecode(res, size))
         return res
 

From noreply at buildbot.pypy.org  Fri Apr 17 10:03:05 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 17 Apr 2015 10:03:05 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof2: reapply vmprof
Message-ID: <20150417080305.E06ED1C1331@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof2
Changeset: r76815:4a9f826c3f32
Date: 2015-04-17 10:02 +0200
http://bitbucket.org/pypy/pypy/changeset/4a9f826c3f32/

Log:	reapply vmprof

diff too long, truncating to 2000 out of 2558 lines

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -9,7 +9,6 @@
 from rpython.rlib.signature import signature
 from rpython.rlib.rarithmetic import r_uint, SHRT_MIN, SHRT_MAX, \
     INT_MIN, INT_MAX, UINT_MAX, USHRT_MAX
-from rpython.rlib.rweaklist import RWeakListMixin
 
 from pypy.interpreter.executioncontext import (ExecutionContext, ActionFlag,
     UserDelAction)
@@ -367,10 +366,6 @@
 
 # ____________________________________________________________
 
-class CodeObjWeakList(RWeakListMixin):
-    def __init__(self):
-        self.initialize()
-
 class ObjSpace(object):
     """Base class for the interpreter-level implementations of object spaces.
     http://pypy.readthedocs.org/en/latest/objspace.html"""
@@ -394,7 +389,6 @@
         self.check_signal_action = None   # changed by the signal module
         self.user_del_action = UserDelAction(self)
         self._code_of_sys_exc_info = None
-        self.all_code_objs = CodeObjWeakList()
 
         # can be overridden to a subclass
         self.initialize()
@@ -672,16 +666,16 @@
             assert ec is not None
             return ec
 
+    def register_code_callback(self, callback):
+        ec = self.getexecutioncontext()
+        ec._code_callback = callback
+
     def register_code_object(self, pycode):
-        callback = self.getexecutioncontext().register_code_callback
-        if callback is not None:
-            callback(self, pycode)
-        self.all_code_objs.add_handle(pycode)
-
-    def set_code_callback(self, callback):
         ec = self.getexecutioncontext()
-        ec.register_code_callback = callback
-        
+        if ec._code_callback is None:
+            return
+        ec._code_callback(self, pycode)
+    
     def _freeze_(self):
         return True
 
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -2,6 +2,7 @@
 from pypy.interpreter.error import OperationError, get_cleared_operation_error
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib import jit
+from rpython.rlib.objectmodel import we_are_translated
 
 TICK_COUNTER_STEP = 100
 
@@ -33,11 +34,16 @@
         self.profilefunc = None
         self.w_profilefuncarg = None
         self.thread_disappeared = False   # might be set to True after os.fork()
-        self.register_code_callback = None
+
         if sys.maxint == 2147483647:
             self._code_unique_id = 0 # XXX this is wrong, it won't work on 32bit
         else:
-            self._code_unique_id = 0x7000000000000000
+            if we_are_translated():
+                self._code_unique_id = 0x7000000000000000
+            else:
+                self._code_unique_id = 0x7700000000000000
+                # should be enough code objects
+        self._code_callback = None
 
     @staticmethod
     def _mark_thread_disappeared(space):
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -129,7 +129,7 @@
 
         ec = self.space.getexecutioncontext()
         self._unique_id = ec._code_unique_id
-        ec._code_unique_id += 2 # so we have one bit that we can mark stuff
+        ec._code_unique_id += 4 # so we have two bits that we can mark stuff
         # with
 
     def _get_full_name(self):
diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -3,13 +3,15 @@
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rtyper.annlowlevel import cast_instance_to_gcref, cast_base_ptr_to_instance
 from rpython.rlib.objectmodel import we_are_translated
-from rpython.rlib import jit, rposix, entrypoint
+from rpython.rlib import jit, rposix, rgc
+from rpython.rlib.rarithmetic import ovfcheck_float_to_int
 from rpython.rtyper.tool import rffi_platform as platform
 from rpython.rlib.rstring import StringBuilder
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import oefmt, wrap_oserror, OperationError
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.pyframe import PyFrame
+from pypy.interpreter.pycode import PyCode
 
 ROOT = py.path.local(__file__).join('..')
 SRC = ROOT.join('src')
@@ -28,14 +30,13 @@
     libraries = ['unwind'],
     
     post_include_bits=["""
-        void* pypy_vmprof_get_virtual_ip(void*);
         void pypy_vmprof_init(void);
     """],
     
     separate_module_sources=["""
         void pypy_vmprof_init(void) {
             vmprof_set_mainloop(pypy_execute_frame_trampoline, 0,
-                                pypy_vmprof_get_virtual_ip);
+                                NULL);
         }
     """],
     )
@@ -56,7 +57,7 @@
 
 pypy_execute_frame_trampoline = rffi.llexternal(
     "pypy_execute_frame_trampoline",
-    [llmemory.GCREF, llmemory.GCREF, llmemory.GCREF],
+    [llmemory.GCREF, llmemory.GCREF, llmemory.GCREF, lltype.Signed],
     llmemory.GCREF,
     compilation_info=eci,
     _nowrapper=True, sandboxsafe=True,
@@ -96,23 +97,15 @@
             gc_frame = cast_instance_to_gcref(frame)
             gc_inputvalue = cast_instance_to_gcref(w_inputvalue)
             gc_operr = cast_instance_to_gcref(operr)
-            gc_result = pypy_execute_frame_trampoline(gc_frame, gc_inputvalue, gc_operr)
+            assert frame.pycode._unique_id & 3 == 0
+            unique_id = frame.pycode._unique_id | 1
+            gc_result = pypy_execute_frame_trampoline(gc_frame, gc_inputvalue,
+                                                      gc_operr, unique_id)
             return cast_base_ptr_to_instance(W_Root, gc_result)
         else:
             return original_execute_frame(frame, w_inputvalue, operr)
 
 
- at entrypoint.entrypoint_lowlevel('main', [llmemory.GCREF],
-                                'pypy_vmprof_get_virtual_ip', True)
-def get_virtual_ip(gc_frame):
-    frame = cast_base_ptr_to_instance(PyFrame, gc_frame)
-    if jit._get_virtualizable_token(frame):
-        return rffi.cast(rffi.VOIDP, 0)
-    virtual_ip = do_get_virtual_ip(frame)
-    return rffi.cast(rffi.VOIDP, virtual_ip)
-
-def do_get_virtual_ip(frame):
-    return frame.pycode._unique_id
 
 def write_long_to_string_builder(l, b):
     if sys.maxint == 2147483647:
@@ -130,31 +123,33 @@
         b.append(chr((l >> 48) & 0xff))
         b.append(chr((l >> 56) & 0xff))
 
+def try_cast_to_pycode(gcref):
+    return rgc.try_cast_gcref_to_instance(PyCode, gcref)
+
+MAX_CODES = 1000
+
 class VMProf(object):
     def __init__(self):
         self.is_enabled = False
         self.ever_enabled = False
-        self.mapping_so_far = [] # stored mapping in between runs
         self.fileno = -1
+        self.current_codes = []
 
-    def enable(self, space, fileno, period):
+    def enable(self, space, fileno, period_usec):
         if self.is_enabled:
             raise oefmt(space.w_ValueError, "_vmprof already enabled")
         self.fileno = fileno
         self.is_enabled = True
-        self.write_header(fileno, period)
+        self.write_header(fileno, period_usec)
         if not self.ever_enabled:
             if we_are_translated():
                 pypy_vmprof_init()
             self.ever_enabled = True
-        for weakcode in space.all_code_objs.get_all_handles():
-            code = weakcode()
-            if code:
-                self.register_code(space, code)
-        space.set_code_callback(vmprof_register_code)
+        self.gather_all_code_objs(space)
+        space.register_code_callback(vmprof_register_code)
         if we_are_translated():
             # does not work untranslated
-            res = vmprof_enable(fileno, period, 0,
+            res = vmprof_enable(fileno, period_usec, 0,
                                 lltype.nullptr(rffi.CCHARP.TO), 0)
         else:
             res = 0
@@ -162,42 +157,55 @@
             raise wrap_oserror(space, OSError(rposix.get_saved_errno(),
                                               "_vmprof.enable"))
 
-    def write_header(self, fileno, period):
-        if period == -1:
-            period_usec = 1000000 / 100 #  100hz
-        else:
-            period_usec = period
+    def gather_all_code_objs(self, space):
+        all_code_objs = rgc.do_get_objects(try_cast_to_pycode)
+        for code in all_code_objs:
+            self.register_code(space, code)
+
+    def write_header(self, fileno, period_usec):
+        assert period_usec > 0
         b = StringBuilder()
         write_long_to_string_builder(0, b)
         write_long_to_string_builder(3, b)
         write_long_to_string_builder(0, b)
         write_long_to_string_builder(period_usec, b)
         write_long_to_string_builder(0, b)
+        b.append('\x04') # interp name
+        b.append(chr(len('pypy')))
+        b.append('pypy')
         os.write(fileno, b.build())
 
     def register_code(self, space, code):
         if self.fileno == -1:
             raise OperationError(space.w_RuntimeError,
                                  space.wrap("vmprof not running"))
-        name = code._get_full_name()
+        self.current_codes.append(code)
+        if len(self.current_codes) >= MAX_CODES:
+            self._flush_codes(space)
+
+    def _flush_codes(self, space):
         b = StringBuilder()
-        b.append('\x02')
-        write_long_to_string_builder(code._unique_id, b)
-        write_long_to_string_builder(len(name), b)
-        b.append(name)
+        for code in self.current_codes:
+            name = code._get_full_name()
+            b.append('\x02')
+            write_long_to_string_builder(code._unique_id, b)
+            write_long_to_string_builder(len(name), b)
+            b.append(name)
         os.write(self.fileno, b.build())
+        self.current_codes = []
 
     def disable(self, space):
         if not self.is_enabled:
             raise oefmt(space.w_ValueError, "_vmprof not enabled")
         self.is_enabled = False
+        space.register_code_callback(None)
+        self._flush_codes(space)
         self.fileno = -1
         if we_are_translated():
            # does not work untranslated
             res = vmprof_disable()
         else:
             res = 0
-        space.set_code_callback(None)
         if res == -1:
             raise wrap_oserror(space, OSError(rposix.get_saved_errno(),
                                               "_vmprof.disable"))
@@ -207,13 +215,23 @@
     mod_vmprof = space.getbuiltinmodule('_vmprof')
     assert isinstance(mod_vmprof, Module)
     mod_vmprof.vmprof.register_code(space, code)
-        
- at unwrap_spec(fileno=int, period=int)
-def enable(space, fileno, period=-1):
+
+ at unwrap_spec(fileno=int, period=float)
+def enable(space, fileno, period=0.01):   # default 100 Hz
     from pypy.module._vmprof import Module
     mod_vmprof = space.getbuiltinmodule('_vmprof')
     assert isinstance(mod_vmprof, Module)
-    mod_vmprof.vmprof.enable(space, fileno, period)
+    #
+    try:
+        period_usec = ovfcheck_float_to_int(period * 1000000.0 + 0.5)
+        if period_usec <= 0 or period_usec >= 1e6:
+            # we don't want seconds here at all
+            raise ValueError
+    except (ValueError, OverflowError):
+        raise OperationError(space.w_ValueError,
+                             space.wrap("'period' too large or non positive"))
+    #
+    mod_vmprof.vmprof.enable(space, fileno, period_usec)
 
 def disable(space):
     from pypy.module._vmprof import Module
diff --git a/pypy/module/_vmprof/src/fake_pypy_api.c b/pypy/module/_vmprof/src/fake_pypy_api.c
--- a/pypy/module/_vmprof/src/fake_pypy_api.c
+++ b/pypy/module/_vmprof/src/fake_pypy_api.c
@@ -1,25 +1,15 @@
-
-long pypy_jit_start_addr(void)
-{
-	return 3;
-}
-
-long pypy_jit_end_addr(void)
-{
-	return 3;
-}
 
 long pypy_jit_stack_depth_at_loc(long x)
 {
 	return 0;
 }
 
-long pypy_find_codemap_at_addr(long x)
+void *pypy_find_codemap_at_addr(long x)
 {
-	return 0;
+	return (void *)0;
 }
 
-long pypy_yield_codemap_at_addr(long x, long y, long *a)
+long pypy_yield_codemap_at_addr(void *x, long y, long *a)
 {
 	return 0;
 }
@@ -27,3 +17,5 @@
 void pypy_pyframe_execute_frame(void)
 {
 }
+
+volatile int pypy_codemap_currently_invalid = 0;
diff --git a/pypy/module/_vmprof/src/get_custom_offset.c b/pypy/module/_vmprof/src/get_custom_offset.c
--- a/pypy/module/_vmprof/src/get_custom_offset.c
+++ b/pypy/module/_vmprof/src/get_custom_offset.c
@@ -1,46 +1,65 @@
 
-long pypy_jit_start_addr();
-long pypy_jit_end_addr();
-long pypy_jit_stack_depth_at_loc(long);
-long pypy_find_codemap_at_addr(long);
-long pypy_yield_codemap_at_addr(long, long, long*);
+extern volatile int pypy_codemap_currently_invalid;
+
+void *pypy_find_codemap_at_addr(long addr, long *start_addr);
+long pypy_yield_codemap_at_addr(void *codemap_raw, long addr,
+                                long *current_pos_addr);
+long pypy_jit_stack_depth_at_loc(long loc);
+
 
 void vmprof_set_tramp_range(void* start, void* end)
 {
 }
 
-static ptrdiff_t vmprof_unw_get_custom_offset(void* ip, unw_cursor_t *cp) {
-	intptr_t ip_l = (intptr_t)ip;
+int custom_sanity_check()
+{
+    return !pypy_codemap_currently_invalid;
+}
 
-	if (ip_l < pypy_jit_start_addr() || ip_l > pypy_jit_end_addr()) {
-		return -1;
-	}
-	return (void*)pypy_jit_stack_depth_at_loc(ip_l);
+static ptrdiff_t vmprof_unw_get_custom_offset(void* ip, void *cp) {
+    intptr_t ip_l = (intptr_t)ip;
+    return pypy_jit_stack_depth_at_loc(ip_l);
 }
 
 static long vmprof_write_header_for_jit_addr(void **result, long n,
-											 void *ip, int max_depth)
+                                             void *ip, int max_depth)
 {
-	long codemap_pos;
-	long current_pos = 0;
-	intptr_t id;
-	intptr_t addr = (intptr_t)ip;
+    void *codemap;
+    long current_pos = 0;
+    intptr_t id;
+    long start_addr = 0;
+    intptr_t addr = (intptr_t)ip;
+    int start, k;
+    void *tmp;
 
-	if (addr < pypy_jit_start_addr() || addr > pypy_jit_end_addr()) {
-		return n;
-	}
-	codemap_pos = pypy_find_codemap_at_addr(addr);
-	if (codemap_pos == -1) {
-		return n;
-	}
-	while (1) {
-		id = pypy_yield_codemap_at_addr(codemap_pos, addr, ¤t_pos);
-		if (id == 0) {
-			return n;
-		}
-		result[n++] = id;
-		if (n >= max_depth) {
-			return n;
-		}
-	}
+    codemap = pypy_find_codemap_at_addr(addr, &start_addr);
+    if (codemap == NULL)
+        // not a jit code at all
+        return n;
+
+    // modify the last entry to point to start address and not the random one
+    // in the middle
+    result[n - 1] = (void*)start_addr;
+    start = n;
+    while (n < max_depth) {
+        id = pypy_yield_codemap_at_addr(codemap, addr, ¤t_pos);
+        if (id == 0)
+            // finish
+            break;
+        result[n++] = (void *)id;
+    }
+    // we strip the topmost part - the reason is that it's either
+    // represented in the jitted caller or it's not jitted (we have the
+    // same function essentially twice
+    k = 0;
+    while (k < (n - start) / 2) {
+        tmp = result[start + k];
+        result[start + k] = result[n - k - 1];
+        result[n - k - 1] = tmp;
+        k++;
+    }
+    if (n != max_depth) {
+        n--;
+    }
+    return n;
 }
diff --git a/pypy/module/_vmprof/src/trampoline.asmgcc.s b/pypy/module/_vmprof/src/trampoline.asmgcc.s
--- a/pypy/module/_vmprof/src/trampoline.asmgcc.s
+++ b/pypy/module/_vmprof/src/trampoline.asmgcc.s
@@ -6,11 +6,10 @@
 	.type	pypy_execute_frame_trampoline, @function
 pypy_execute_frame_trampoline:
 	.cfi_startproc
-	pushq	%rdi
+	pushq	%rcx
 	.cfi_def_cfa_offset 16
 	call pypy_pyframe_execute_frame at PLT
-	/* GCROOT 0(%rsp) */
-	popq	%rdi
+	popq	%rcx
 	.cfi_def_cfa_offset 8
 	ret
 	.cfi_endproc
diff --git a/pypy/module/_vmprof/src/trampoline.h b/pypy/module/_vmprof/src/trampoline.h
--- a/pypy/module/_vmprof/src/trampoline.h
+++ b/pypy/module/_vmprof/src/trampoline.h
@@ -1,1 +1,1 @@
-void* pypy_execute_frame_trampoline(void*, void*, void*);
+void* pypy_execute_frame_trampoline(void*, void*, void*, long);
diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c
--- a/pypy/module/_vmprof/src/vmprof.c
+++ b/pypy/module/_vmprof/src/vmprof.c
@@ -25,6 +25,8 @@
 #include 
 #include 
 #include 
+#include 
+#include 
 
 #define UNW_LOCAL_ONLY
 #include 
@@ -34,12 +36,18 @@
 #define _unused(x) ((void)x)
 
 #define MAX_FUNC_NAME 128
-#define MAX_STACK_DEPTH 64
+#define MAX_STACK_DEPTH 1024
+#define BUFFER_SIZE 8192
 
-static FILE* profile_file = NULL;
+
+static int profile_file = 0;
+static char profile_write_buffer[BUFFER_SIZE];
+static int profile_buffer_position = 0;
 void* vmprof_mainloop_func;
 static ptrdiff_t mainloop_sp_offset;
 static vmprof_get_virtual_ip_t mainloop_get_virtual_ip;
+static long last_period_usec = 0;
+static int atfork_hook_installed = 0;
 
 
 /* *************************************************************
@@ -51,27 +59,33 @@
 #define MARKER_VIRTUAL_IP '\x02'
 #define MARKER_TRAILER '\x03'
 
-static void prof_word(FILE* f, long x) {
-    fwrite(&x, sizeof(x), 1, f);
+static void prof_word(long x) {
+	((long*)(profile_write_buffer + profile_buffer_position))[0] = x;
+	profile_buffer_position += sizeof(long);
 }
 
-static void prof_header(FILE* f, long period_usec) {
-    prof_word(f, 0);
-    prof_word(f, 3);
-    prof_word(f, 0);
-    prof_word(f, period_usec);
-    prof_word(f, 0);
+static void prof_header(long period_usec) {
+    // XXX never used here?
+    prof_word(0);
+    prof_word(3);
+    prof_word(0);
+    prof_word(period_usec);
+    prof_word(0);
+    write(profile_file, profile_write_buffer, profile_buffer_position);
+    profile_buffer_position = 0;
 }
 
-static void prof_write_stacktrace(FILE* f, void** stack, int depth, int count) {
+static void prof_write_stacktrace(void** stack, int depth, int count) {
     int i;
 	char marker = MARKER_STACKTRACE;
 
-	fwrite(&marker, 1, 1, f);
-    prof_word(f, count);
-    prof_word(f, depth);
+	profile_write_buffer[profile_buffer_position++] = MARKER_STACKTRACE;
+    prof_word(count);
+    prof_word(depth);
     for(i=0; i= 0);
     _unused(ret);
+	int first_run = 1;
 
     while (n < max_depth) {
         if (unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *) &ip) < 0) {
@@ -173,16 +192,21 @@
           void **arg_ptr = (void**)arg_addr;
           // fprintf(stderr, "stacktrace mainloop: rsp %p   &f2 %p   offset %ld\n", 
           //         sp, arg_addr, mainloop_sp_offset);
-          ip = mainloop_get_virtual_ip(*arg_ptr);
+		  if (mainloop_get_virtual_ip) {
+			  ip = mainloop_get_virtual_ip(*arg_ptr);
+		  } else {
+			  ip = *arg_ptr;
+		  }
         }
 
         result[n++] = ip;
 		n = vmprof_write_header_for_jit_addr(result, n, ip, max_depth);
-        if (vmprof_unw_step(&cursor) <= 0) {
+        if (vmprof_unw_step(&cursor, first_run) <= 0) {
             break;
         }
+		first_run = 0;
     }
-    --recursive;
+    //--recursive;
     return n;
 }
 
@@ -193,10 +217,12 @@
 
 static void sigprof_handler(int sig_nr, siginfo_t* info, void *ucontext) {
     void* stack[MAX_STACK_DEPTH];
+    int saved_errno = errno;
     stack[0] = GetPC((ucontext_t*)ucontext);
     int depth = frame_forcer(get_stack_trace(stack+1, MAX_STACK_DEPTH-1, ucontext));
     depth++;  // To account for pc value in stack[0];
-    prof_write_stacktrace(profile_file, stack, depth, 1);
+    prof_write_stacktrace(stack, depth, 1);
+    errno = saved_errno;
 }
 
 /* *************************************************************
@@ -209,14 +235,12 @@
 	if ((fd = dup(fd)) == -1) {
 		return -1;
 	}
-    profile_file = fdopen(fd, "wb");
-	if (!profile_file) {
-		return -1;
-	}
+	profile_buffer_position = 0;
+    profile_file = fd;
 	if (write_header)
-		prof_header(profile_file, period_usec);
+		prof_header(period_usec);
 	if (s)
-		fwrite(s, slen, 1, profile_file);
+		write(profile_file, s, slen);
 	return 0;
 }
 
@@ -226,16 +250,16 @@
     char buf[BUFSIZ];
     size_t size;
 	int marker = MARKER_TRAILER;
-	fwrite(&marker, 1, 1, profile_file);
+	write(profile_file, &marker, 1);
 
     // copy /proc/PID/maps to the end of the profile file
     sprintf(buf, "/proc/%d/maps", getpid());
     src = fopen(buf, "r");    
     while ((size = fread(buf, 1, BUFSIZ, src))) {
-        fwrite(buf, 1, size, profile_file);
+        write(profile_file, buf, size);
     }
     fclose(src);
-    fclose(profile_file);
+    close(profile_file);
 	return 0;
 }
 
@@ -253,15 +277,16 @@
 }
 
 static int remove_sigprof_handler(void) {
-    //sighandler_t res = signal(SIGPROF, SIG_DFL);
-	//if (res == SIG_ERR) {
-	//	return -1;
-	//}
+    sighandler_t res = signal(SIGPROF, SIG_DFL);
+	if (res == SIG_ERR) {
+		return -1;
+	}
 	return 0;
 };
 
 static int install_sigprof_timer(long period_usec) {
     static struct itimerval timer;
+    last_period_usec = period_usec;
     timer.it_interval.tv_sec = 0;
     timer.it_interval.tv_usec = period_usec;
     timer.it_value = timer.it_interval;
@@ -273,15 +298,45 @@
 
 static int remove_sigprof_timer(void) {
     static struct itimerval timer;
+    last_period_usec = 0;
     timer.it_interval.tv_sec = 0;
     timer.it_interval.tv_usec = 0;
-    timer.it_value = timer.it_interval;
+    timer.it_value.tv_sec = 0;
+    timer.it_value.tv_usec = 0;
     if (setitimer(ITIMER_PROF, &timer, NULL) != 0) {
 		return -1;
     }
 	return 0;
 }
 
+static void atfork_disable_timer(void) {
+    remove_sigprof_timer();
+}
+
+static void atfork_enable_timer(void) {
+    install_sigprof_timer(last_period_usec);
+}
+
+static int install_pthread_atfork_hooks(void) {
+    /* this is needed to prevent the problems described there:
+         - http://code.google.com/p/gperftools/issues/detail?id=278
+         - http://lists.debian.org/debian-glibc/2010/03/msg00161.html
+
+        TL;DR: if the RSS of the process is large enough, the clone() syscall
+        will be interrupted by the SIGPROF before it can complete, then
+        retried, interrupted again and so on, in an endless loop.  The
+        solution is to disable the timer around the fork, and re-enable it
+        only inside the parent.
+    */
+    if (atfork_hook_installed)
+        return 0;
+    int ret = pthread_atfork(atfork_disable_timer, atfork_enable_timer, NULL);
+    if (ret != 0)
+        return -1;
+    atfork_hook_installed = 1;
+    return 0;
+}
+
 /* *************************************************************
  * public API
  * *************************************************************
@@ -297,8 +352,7 @@
 int vmprof_enable(int fd, long period_usec, int write_header, char *s,
 				  int slen)
 {
-    if (period_usec == -1)
-        period_usec = 1000000 / 100; /* 100hz */
+    assert(period_usec > 0);
     if (open_profile(fd, period_usec, write_header, s, slen) == -1) {
 		return -1;
 	}
@@ -308,6 +362,9 @@
     if (install_sigprof_timer(period_usec) == -1) {
 		return -1;
 	}
+    if (install_pthread_atfork_hooks() == -1) {
+        return -1;
+    }
 	return 0;
 }
 
@@ -325,6 +382,7 @@
 }
 
 void vmprof_register_virtual_function(const char* name, void* start, void* end) {
+	// XXX unused by pypy
     // for now *end is simply ignored
 	char buf[1024];
 	int lgt = strlen(name) + 2 * sizeof(long) + 1;
@@ -336,5 +394,5 @@
 	((void **)(((void*)buf) + 1))[0] = start;
 	((long *)(((void*)buf) + 1 + sizeof(long)))[0] = lgt - 2 * sizeof(long) - 1;
 	strncpy(buf + 2 * sizeof(long) + 1, name, 1024 - 2 * sizeof(long) - 1);
-	fwrite(buf, lgt, 1, profile_file);
+	write(profile_file, buf, lgt);
 }
diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py
--- a/pypy/module/_vmprof/test/test__vmprof.py
+++ b/pypy/module/_vmprof/test/test__vmprof.py
@@ -21,6 +21,11 @@
             i = 0
             count = 0
             i += 5 * WORD # header
+            assert s[i] == '\x04'
+            i += 1 # marker
+            assert s[i] == '\x04'
+            i += 1 # length
+            i += len('pypy')
             while i < len(s):
                 assert s[i] == '\x02'
                 i += 1
@@ -53,3 +58,11 @@
         assert "py:foo:" in s
         assert "py:foo2:" in s
         assert no_of_codes2 >= no_of_codes + 2 # some extra codes from tests
+
+    def test_enable_ovf(self):
+        import _vmprof
+        raises(ValueError, _vmprof.enable, 999, 0)
+        raises(ValueError, _vmprof.enable, 999, -2.5)
+        raises(ValueError, _vmprof.enable, 999, 1e300)
+        raises(ValueError, _vmprof.enable, 999, 1e300 * 1e300)
+        raises(ValueError, _vmprof.enable, 999, (1e300*1e300) / (1e300*1e300))
diff --git a/pypy/module/_vmprof/test/test_direct.py b/pypy/module/_vmprof/test/test_direct.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/test/test_direct.py
@@ -0,0 +1,66 @@
+
+import cffi, py
+
+srcdir = py.path.local(__file__).join("..", "..", "src")
+
+ffi = cffi.FFI()
+ffi.cdef("""
+long vmprof_write_header_for_jit_addr(void **, long, void*, int);
+void *pypy_find_codemap_at_addr(long addr, long *start_addr);
+long pypy_yield_codemap_at_addr(void *codemap_raw, long addr,
+                                long *current_pos_addr);
+long buffer[];
+""")
+
+lib = ffi.verify("""
+volatile int pypy_codemap_currently_invalid = 0;
+
+long buffer[] = {0, 0, 0, 0, 0};
+
+
+
+void *pypy_find_codemap_at_addr(long addr, long *start_addr)
+{
+    return (void*)buffer;
+}
+
+long pypy_yield_codemap_at_addr(void *codemap_raw, long addr,
+                                long *current_pos_addr)
+{
+    long c = *current_pos_addr;
+    if (c >= 5)
+        return 0;
+    *current_pos_addr = c + 1;
+    return *((long*)codemap_raw + c);
+}
+
+
+""" + open(str(srcdir.join("get_custom_offset.c"))).read())
+
+class TestDirect(object):
+    def test_infrastructure(self):
+        cont = ffi.new("long[1]", [0])
+        buf = lib.pypy_find_codemap_at_addr(0, cont)
+        assert buf
+        cont[0] = 0
+        next_addr = lib.pypy_yield_codemap_at_addr(buf, 0, cont)
+        assert cont[0] == 1
+        assert not next_addr
+        lib.buffer[0] = 13
+        cont[0] = 0
+        next_addr = lib.pypy_yield_codemap_at_addr(buf, 0, cont)
+        assert int(ffi.cast("long", next_addr)) == 13
+
+    def test_write_header_for_jit_addr(self):
+        lib.buffer[0] = 4
+        lib.buffer[1] = 8
+        lib.buffer[2] = 12
+        lib.buffer[3] = 16
+        lib.buffer[4] = 0
+        buf = ffi.new("long[5]", [0] * 5)
+        result = ffi.cast("void**", buf)
+        res = lib.vmprof_write_header_for_jit_addr(result, 0, ffi.NULL, 100)
+        assert res == 3
+        assert buf[0] == 16
+        assert buf[1] == 12
+        assert buf[2] == 8
diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py
--- a/pypy/module/gc/referents.py
+++ b/pypy/module/gc/referents.py
@@ -44,30 +44,6 @@
     return OperationError(space.w_NotImplementedError,
                           space.wrap("operation not implemented by this GC"))
 
-# ____________________________________________________________
-
-def clear_gcflag_extra(fromlist):
-    pending = fromlist[:]
-    while pending:
-        gcref = pending.pop()
-        if rgc.get_gcflag_extra(gcref):
-            rgc.toggle_gcflag_extra(gcref)
-            pending.extend(rgc.get_rpy_referents(gcref))
-
-def do_get_objects():
-    roots = [gcref for gcref in rgc.get_rpy_roots() if gcref]
-    pending = roots[:]
-    result_w = []
-    while pending:
-        gcref = pending.pop()
-        if not rgc.get_gcflag_extra(gcref):
-            rgc.toggle_gcflag_extra(gcref)
-            w_obj = try_cast_gcref_to_w_root(gcref)
-            if w_obj is not None:
-                result_w.append(w_obj)
-            pending.extend(rgc.get_rpy_referents(gcref))
-    clear_gcflag_extra(roots)
-    return result_w
 
 # ____________________________________________________________
 
@@ -116,8 +92,8 @@
                 break
     # done.  Clear flags carefully
     rgc.toggle_gcflag_extra(gcarg)
-    clear_gcflag_extra(roots)
-    clear_gcflag_extra([gcarg])
+    rgc.clear_gcflag_extra(roots)
+    rgc.clear_gcflag_extra([gcarg])
     return result_w
 
 # ____________________________________________________________
@@ -189,8 +165,7 @@
     """Return a list of all app-level objects."""
     if not rgc.has_gcflag_extra():
         raise missing_operation(space)
-    result_w = do_get_objects()
-    rgc.assert_no_more_gcflags()
+    result_w = rgc.do_get_objects(try_cast_gcref_to_w_root)
     return space.newlist(result_w)
 
 def get_referents(space, args_w):
diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py
--- a/pypy/module/pypyjit/interp_resop.py
+++ b/pypy/module/pypyjit/interp_resop.py
@@ -105,7 +105,7 @@
             ofs = ops_offset.get(op, 0)
         if op.opnum == rop.DEBUG_MERGE_POINT:
             jd_sd = jitdrivers_sd[op.getarg(0).getint()]
-            greenkey = op.getarglist()[3:]
+            greenkey = op.getarglist()[4:]
             repr = jd_sd.warmstate.get_location_str(greenkey)
             w_greenkey = wrap_greenkey(space, jd_sd.jitdriver, greenkey, repr)
             l_w.append(DebugMergePoint(space, jit_hooks._cast_to_gcref(op),
diff --git a/pypy/module/pypyjit/test/test_jit_hook.py b/pypy/module/pypyjit/test/test_jit_hook.py
--- a/pypy/module/pypyjit/test/test_jit_hook.py
+++ b/pypy/module/pypyjit/test/test_jit_hook.py
@@ -55,7 +55,7 @@
         oplist = parse("""
         [i1, i2, p2]
         i3 = int_add(i1, i2)
-        debug_merge_point(0, 0, 0, 0, 0, ConstPtr(ptr0))
+        debug_merge_point(0, 0, 0, 0, 0, 0, ConstPtr(ptr0))
         guard_nonnull(p2) []
         guard_true(i3) []
         """, namespace={'ptr0': code_gcref}).operations
diff --git a/rpython/bin/rpython-vmprof b/rpython/bin/rpython-vmprof
new file mode 100755
--- /dev/null
+++ b/rpython/bin/rpython-vmprof
@@ -0,0 +1,28 @@
+#!/usr/bin/env pypy
+
+"""RPython translation usage:
+
+rpython  target 
+
+run with --help for more information
+"""
+
+import sys, os
+sys.path.insert(0, os.path.dirname(os.path.dirname(
+                       os.path.dirname(os.path.realpath(__file__)))))
+from rpython.translator.goal.translate import main
+
+# no implicit targets
+if len(sys.argv) == 1:
+    print __doc__
+    sys.exit(1)
+
+import _vmprof, subprocess
+x = subprocess.Popen('gzip > vmprof.log.gz', shell=True, stdin=subprocess.PIPE)
+_vmprof.enable(x.stdin.fileno(), 0.001)
+try:
+    main()
+finally:
+    _vmprof.disable()
+    x.stdin.close()
+    x.wait()
diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -102,7 +102,7 @@
         self.store_reg(mc, r.r0, r.fp, ofs)
         mc.MOV_rr(r.r0.value, r.fp.value)
         self.gen_func_epilog(mc)
-        rawstart = mc.materialize(self.cpu.asmmemmgr, [])
+        rawstart = mc.materialize(self.cpu, [])
         self.propagate_exception_path = rawstart
 
     def _store_and_reset_exception(self, mc, excvalloc=None, exctploc=None,
@@ -198,7 +198,7 @@
         mc.ADD_ri(r.sp.value, r.sp.value, (len(r.argument_regs) + 2) * WORD)
         mc.B(self.propagate_exception_path)
         #
-        rawstart = mc.materialize(self.cpu.asmmemmgr, [])
+        rawstart = mc.materialize(self.cpu, [])
         self.stack_check_slowpath = rawstart
 
     def _build_wb_slowpath(self, withcards, withfloats=False, for_frame=False):
@@ -255,7 +255,7 @@
         #
         mc.POP([r.ip.value, r.pc.value])
         #
-        rawstart = mc.materialize(self.cpu.asmmemmgr, [])
+        rawstart = mc.materialize(self.cpu, [])
         if for_frame:
             self.wb_slowpath[4] = rawstart
         else:
@@ -276,7 +276,7 @@
                                       callee_only)
         # return
         mc.POP([r.ip.value, r.pc.value])
-        return mc.materialize(self.cpu.asmmemmgr, [])
+        return mc.materialize(self.cpu, [])
 
     def _build_malloc_slowpath(self, kind):
         """ While arriving on slowpath, we have a gcpattern on stack 0.
@@ -352,7 +352,7 @@
         mc.POP([r.ip.value, r.pc.value])
 
         #
-        rawstart = mc.materialize(self.cpu.asmmemmgr, [])
+        rawstart = mc.materialize(self.cpu, [])
         return rawstart
 
     def _reload_frame_if_necessary(self, mc):
@@ -473,7 +473,7 @@
         mc.MOV_rr(r.r0.value, r.fp.value)
         #
         self.gen_func_epilog(mc)
-        rawstart = mc.materialize(self.cpu.asmmemmgr, [])
+        rawstart = mc.materialize(self.cpu, [])
         self.failure_recovery_code[exc + 2 * withfloats] = rawstart
 
     def generate_quick_failure(self, guardtok):
@@ -851,7 +851,7 @@
         # restore registers
         self._pop_all_regs_from_jitframe(mc, [], self.cpu.supports_floats)
         mc.POP([r.ip.value, r.pc.value])  # return
-        self._frame_realloc_slowpath = mc.materialize(self.cpu.asmmemmgr, [])
+        self._frame_realloc_slowpath = mc.materialize(self.cpu, [])
 
     def _load_shadowstack_top(self, mc, reg, gcrootmap):
         rst = gcrootmap.get_root_stack_top_addr()
@@ -881,7 +881,7 @@
         self.datablockwrapper = None
         allblocks = self.get_asmmemmgr_blocks(looptoken)
         size = self.mc.get_relative_pos() 
-        res = self.mc.materialize(self.cpu.asmmemmgr, allblocks,
+        res = self.mc.materialize(self.cpu, allblocks,
                                    self.cpu.gc_ll_descr.gcrootmap)
         self.cpu.codemap.register_codemap(
             self.codemap.get_final_bytecode(res, size))
diff --git a/rpython/jit/backend/arm/runner.py b/rpython/jit/backend/arm/runner.py
--- a/rpython/jit/backend/arm/runner.py
+++ b/rpython/jit/backend/arm/runner.py
@@ -50,6 +50,7 @@
     def setup_once(self):
         self.cpuinfo.arch_version = detect_arch_version()
         self.cpuinfo.hf_abi = detect_hardfloat()
+        self.codemap.setup()
         self.assembler.setup_once()
 
     def finish_once(self):
diff --git a/rpython/jit/backend/arm/test/support.py b/rpython/jit/backend/arm/test/support.py
--- a/rpython/jit/backend/arm/test/support.py
+++ b/rpython/jit/backend/arm/test/support.py
@@ -24,7 +24,7 @@
 
 def run_asm(asm):
     BOOTSTRAP_TP = lltype.FuncType([], lltype.Signed)
-    addr = asm.mc.materialize(asm.cpu.asmmemmgr, [], None)
+    addr = asm.mc.materialize(asm.cpu, [], None)
     assert addr % 8 == 0
     func = rffi.cast(lltype.Ptr(BOOTSTRAP_TP), addr)
     asm.mc._dump_trace(addr, 'test.asm')
diff --git a/rpython/jit/backend/arm/test/test_calling_convention.py b/rpython/jit/backend/arm/test/test_calling_convention.py
--- a/rpython/jit/backend/arm/test/test_calling_convention.py
+++ b/rpython/jit/backend/arm/test/test_calling_convention.py
@@ -29,7 +29,7 @@
         mc = InstrBuilder()
         mc.MOV_rr(r.r0.value, r.sp.value)
         mc.MOV_rr(r.pc.value, r.lr.value)
-        return mc.materialize(self.cpu.asmmemmgr, [])
+        return mc.materialize(self.cpu, [])
 
     def get_alignment_requirements(self):
         return 8
diff --git a/rpython/jit/backend/llsupport/asmmemmgr.py b/rpython/jit/backend/llsupport/asmmemmgr.py
--- a/rpython/jit/backend/llsupport/asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/asmmemmgr.py
@@ -5,9 +5,6 @@
 from rpython.rlib.debug import debug_start, debug_print, debug_stop
 from rpython.rlib.debug import have_debug_prints
 from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rlib.rbisect import bisect, bisect_tuple
-
-_memmngr = None # global reference so we can use @entrypoint :/
 
 
 class AsmMemoryManager(object):
@@ -27,12 +24,6 @@
         self.free_blocks = {}      # map {start: stop}
         self.free_blocks_end = {}  # map {stop: start}
         self.blocks_by_size = [[] for i in range(self.num_indices)]
-        # two lists of jit addresses (sorted) and the corresponding stack
-        # depths
-        self.jit_addr_map = []
-        self.jit_frame_depth_map = []
-        self.jit_codemap = []
-        # see codemap.py
 
     def malloc(self, minsize, maxsize):
         """Allocate executable memory, between minsize and maxsize bytes,
@@ -54,13 +45,6 @@
         if r_uint is not None:
             self.total_mallocs -= r_uint(stop - start)
         self._add_free_block(start, stop)
-        # fix up jit_addr_map
-        jit_adr_start = bisect(self.jit_addr_map, start)
-        jit_adr_stop = bisect(self.jit_addr_map, stop)
-        self.jit_addr_map = (self.jit_addr_map[:jit_adr_start] +
-                             self.jit_addr_map[jit_adr_stop:])
-        self.jit_frame_depth_map = (self.jit_frame_depth_map[:jit_adr_start] +
-                                    self.jit_frame_depth_map[jit_adr_stop:])
 
     def open_malloc(self, minsize):
         """Allocate at least minsize bytes.  Returns (start, stop)."""
@@ -167,35 +151,6 @@
         del self.free_blocks_end[stop]
         return (start, stop)
 
-    def register_frame_depth_map(self, rawstart, frame_positions,
-                                 frame_assignments):
-        if not frame_positions:
-            return
-        if not self.jit_addr_map or rawstart > self.jit_addr_map[-1]:
-            start = len(self.jit_addr_map)
-            self.jit_addr_map += [0] * len(frame_positions)
-            self.jit_frame_depth_map += [0] * len(frame_positions)
-        else:
-            start = bisect(self.jit_addr_map, rawstart)
-            self.jit_addr_map = (self.jit_addr_map[:start] +
-                                 [0] * len(frame_positions) +
-                                 self.jit_addr_map[start:])
-            self.jit_frame_depth_map = (self.jit_frame_depth_map[:start] +
-                                 [0] * len(frame_positions) +
-                                 self.jit_frame_depth_map[start:])
-        for i, pos in enumerate(frame_positions):
-            self.jit_addr_map[i + start] = pos + rawstart
-            self.jit_frame_depth_map[i + start] = frame_assignments[i]
-
-    def register_codemap(self, codemap):
-        start = codemap[0]
-        pos = bisect_tuple(self.jit_codemap, start)
-        if pos == len(self.jit_codemap): # common case
-            self.jit_codemap.append(codemap)
-        else:
-            self.jit_codemap = (self.jit_codemap[:pos] + [codemap] +
-                                self.jit_codemap[pos:])
-
     def _delete(self):
         "NOT_RPYTHON"
         if self._allocated:
@@ -351,11 +306,11 @@
             #
         debug_stop(logname)
 
-    def materialize(self, asmmemmgr, allblocks, gcrootmap=None):
+    def materialize(self, cpu, allblocks, gcrootmap=None):
         size = self.get_relative_pos()
         align = self.ALIGN_MATERIALIZE
         size += align - 1
-        malloced = asmmemmgr.malloc(size, size)
+        malloced = cpu.asmmemmgr.malloc(size, size)
         allblocks.append(malloced)
         rawstart = malloced[0]
         rawstart = (rawstart + align - 1) & (-align)
@@ -364,8 +319,9 @@
             assert gcrootmap is not None
             for pos, mark in self.gcroot_markers:
                 gcrootmap.register_asm_addr(rawstart + pos, mark)
-        asmmemmgr.register_frame_depth_map(rawstart, self.frame_positions,
-                                           self.frame_assignments)
+        cpu.codemap.register_frame_depth_map(rawstart, rawstart + size,
+                                             self.frame_positions,
+                                             self.frame_assignments)
         self.frame_positions = None
         self.frame_assignments = None
         return rawstart
diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -130,7 +130,7 @@
         self.gcmap_for_finish[0] = r_uint(1)
 
     def setup(self, looptoken):
-        self.codemap = CodemapBuilder()
+        self.codemap_builder = CodemapBuilder()
         self._finish_gcmap = lltype.nullptr(jitframe.GCMAP)
 
     def set_debug(self, v):
@@ -200,7 +200,9 @@
         return fail_descr, target
 
     def debug_merge_point(self, op):
-        self.codemap.debug_merge_point(op, self.mc.get_relative_pos())
+        self.codemap_builder.debug_merge_point(op.getarg(1).getint(),
+                                               op.getarg(3).getint(),
+                                               self.mc.get_relative_pos())
 
     def call_assembler(self, op, guard_op, argloc, vloc, result_loc, tmploc):
         self._store_force_index(guard_op)
diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py
--- a/rpython/jit/backend/llsupport/codemap.py
+++ b/rpython/jit/backend/llsupport/codemap.py
@@ -9,79 +9,129 @@
 
 """
 
+import os
 from rpython.rlib import rgc
+from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.rlib.entrypoint import jit_entrypoint
-from rpython.jit.backend.llsupport import asmmemmgr
-from rpython.rlib.rbisect import bisect, bisect_tuple
+from rpython.rlib.rbisect import bisect_right, bisect_right_addr
+from rpython.rlib.rbisect import bisect_left, bisect_left_addr
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+from rpython.translator import cdir
 
- at jit_entrypoint([lltype.Signed], lltype.Signed,
-                c_name='pypy_jit_stack_depth_at_loc')
- at rgc.no_collect
-def stack_depth_at_loc(loc):
-    _memmngr = asmmemmgr._memmngr
 
-    pos = bisect(_memmngr.jit_addr_map, loc)
-    if pos == 0 or pos == len(_memmngr.jit_addr_map):
-        return -1
-    return _memmngr.jit_frame_depth_map[pos-1]
+INT_LIST_PTR = rffi.CArrayPtr(lltype.Signed)
 
- at jit_entrypoint([], lltype.Signed, c_name='pypy_jit_start_addr')
-def jit_start_addr():
-    _memmngr = asmmemmgr._memmngr
 
-    return _memmngr.jit_addr_map[0]
+srcdir = os.path.join(os.path.dirname(__file__), 'src')
 
- at jit_entrypoint([], lltype.Signed, c_name='pypy_jit_end_addr')
-def jit_end_addr():
-    _memmngr = asmmemmgr._memmngr
+eci = ExternalCompilationInfo(post_include_bits=["""
+#include 
+RPY_EXTERN long pypy_jit_codemap_add(uintptr_t addr,
+                                     unsigned int machine_code_size,
+                                     long *bytecode_info,
+                                     unsigned int bytecode_info_size);
+RPY_EXTERN long *pypy_jit_codemap_del(uintptr_t addr);
+RPY_EXTERN uintptr_t pypy_jit_codemap_firstkey(void);
+RPY_EXTERN void *pypy_find_codemap_at_addr(long addr, long* start_addr);
+RPY_EXTERN long pypy_yield_codemap_at_addr(void *codemap_raw, long addr,
+                                           long *current_pos_addr);
 
-    return _memmngr.jit_addr_map[-1]
+RPY_EXTERN long pypy_jit_depthmap_add(uintptr_t addr, unsigned int size,
+                                      unsigned int stackdepth);
+RPY_EXTERN void pypy_jit_depthmap_clear(uintptr_t addr, unsigned int size);
 
- at jit_entrypoint([lltype.Signed], lltype.Signed,
-                c_name='pypy_find_codemap_at_addr')
-def find_codemap_at_addr(addr):
-    _memmngr = asmmemmgr._memmngr
+"""], separate_module_sources=[
+    open(os.path.join(srcdir, 'skiplist.c'), 'r').read() +
+    open(os.path.join(srcdir, 'codemap.c'), 'r').read()
+], include_dirs=[cdir])
 
-    res = bisect_tuple(_memmngr.jit_codemap, addr) - 1
-    if res == len(_memmngr.jit_codemap):
-        return -1
-    return res
+def llexternal(name, args, res):
+    return rffi.llexternal(name, args, res, compilation_info=eci,
+                           releasegil=False)
 
- at jit_entrypoint([lltype.Signed, lltype.Signed,
-                 rffi.CArrayPtr(lltype.Signed)], lltype.Signed,
-                 c_name='pypy_yield_codemap_at_addr')
-def yield_bytecode_at_addr(codemap_no, addr, current_pos_addr):
-    """ will return consecutive unique_ids from codemap, starting from position
-    `pos` until addr
+pypy_jit_codemap_add = llexternal('pypy_jit_codemap_add',
+                                  [lltype.Signed, lltype.Signed,
+                                   INT_LIST_PTR, lltype.Signed],
+                                  lltype.Signed)
+pypy_jit_codemap_del = llexternal('pypy_jit_codemap_del',
+                                  [lltype.Signed], INT_LIST_PTR)
+pypy_jit_codemap_firstkey = llexternal('pypy_jit_codemap_firstkey',
+                                       [], lltype.Signed)
+
+pypy_jit_depthmap_add = llexternal('pypy_jit_depthmap_add',
+                                   [lltype.Signed, lltype.Signed,
+                                    lltype.Signed], lltype.Signed)
+pypy_jit_depthmap_clear = llexternal('pypy_jit_depthmap_clear',
+                                     [lltype.Signed, lltype.Signed],
+                                     lltype.Void)
+
+stack_depth_at_loc = llexternal('pypy_jit_stack_depth_at_loc',
+                                [lltype.Signed], lltype.Signed)
+find_codemap_at_addr = llexternal('pypy_find_codemap_at_addr',
+                                  [lltype.Signed, rffi.CArrayPtr(lltype.Signed)], lltype.Signed)
+yield_bytecode_at_addr = llexternal('pypy_yield_codemap_at_addr',
+                                    [lltype.Signed, lltype.Signed,
+                                     rffi.CArrayPtr(lltype.Signed)],
+                                     lltype.Signed)
+
+
+class CodemapStorage(object):
+    """ An immortal wrapper around underlaying jit codemap data
     """
-    _memmngr = asmmemmgr._memmngr
+    def setup(self):
+        if not we_are_translated():
+             # in case someone failed to call free(), in tests only anyway
+             self.free()
 
-    codemap = _memmngr.jit_codemap[codemap_no]
-    current_pos = current_pos_addr[0]
-    start_addr = codemap[0]
-    rel_addr = addr - start_addr
-    while True:
-        if current_pos >= len(codemap[2]):
-            return 0
-        next_start = codemap[2][current_pos + 1]
-        if next_start > rel_addr:
-            return 0
-        next_stop = codemap[2][current_pos + 2]
-        if next_stop > rel_addr:
-            current_pos_addr[0] = current_pos + 4
-            return codemap[2][current_pos]
-        # we need to skip potentially more than one
-        current_pos = codemap[2][current_pos + 3]
+    def free(self):
+        while True:
+            key = pypy_jit_codemap_firstkey()
+            if not key:
+                break
+            items = pypy_jit_codemap_del(key)
+            lltype.free(items, flavor='raw', track_allocation=False)
+
+    def free_asm_block(self, start, stop):
+        items = pypy_jit_codemap_del(start)
+        if items:
+            lltype.free(items, flavor='raw', track_allocation=False)
+        pypy_jit_depthmap_clear(start, stop - start)
+
+    def register_frame_depth_map(self, rawstart, rawstop, frame_positions,
+                                 frame_assignments):
+        if not frame_positions:
+            return
+        assert len(frame_positions) == len(frame_assignments)
+        for i in range(len(frame_positions)-1, -1, -1):
+            pos = rawstart + frame_positions[i]
+            length = rawstop - pos
+            if length > 0:
+                #print "ADD:", pos, length, frame_assignments[i]
+                pypy_jit_depthmap_add(pos, length, frame_assignments[i])
+            rawstop = pos
+
+    def register_codemap(self, (start, size, l)):
+        items = lltype.malloc(INT_LIST_PTR.TO, len(l), flavor='raw',
+                              track_allocation=False)
+        for i in range(len(l)):
+            items[i] = l[i]
+        if pypy_jit_codemap_add(start, size, items, len(l)) < 0:
+            lltype.free(items, flavor='raw', track_allocation=False)
+
+    def finish_once(self):
+        self.free()
 
 def unpack_traceback(addr):
-    codemap_pos = find_codemap_at_addr(addr)
-    assert codemap_pos >= 0
+    codemap_raw = find_codemap_at_addr(addr,
+                                lltype.nullptr(rffi.CArray(lltype.Signed)))
+    if not codemap_raw:
+        return [] # no codemap for that position
     storage = lltype.malloc(rffi.CArray(lltype.Signed), 1, flavor='raw')
     storage[0] = 0
     res = []
     while True:
-        item = yield_bytecode_at_addr(codemap_pos, addr, storage)
+        item = yield_bytecode_at_addr(codemap_raw, addr, storage)
         if item == 0:
             break
         res.append(item)
@@ -95,14 +145,18 @@
         self.patch_position = []
         self.last_call_depth = -1
 
-    def debug_merge_point(self, op, pos):
-        call_depth = op.getarg(1).getint()
+    def debug_merge_point(self, call_depth, unique_id, pos):
         if call_depth != self.last_call_depth:
-            unique_id = op.getarg(3).getint()
             if unique_id == 0: # uninteresting case
                 return
             assert unique_id & 1 == 0
             if call_depth > self.last_call_depth:
+                assert call_depth == self.last_call_depth + 1
+                # ^^^ It should never be the case that we see
+                # debug_merge_points that suddenly go more than *one*
+                # call deeper than the previous one (unless we're at
+                # the start of a bridge, handled by
+                # inherit_code_from_position()).
                 self.l.append(unique_id)
                 self.l.append(pos) # <- this is a relative pos
                 self.patch_position.append(len(self.l))
@@ -139,4 +193,3 @@
             item = self.l[i * 4 + 3] # end in l
             assert item > 0
         return (addr, size, self.l) # XXX compact self.l
-
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -16,7 +16,7 @@
     FieldDescr, ArrayDescr, CallDescr, InteriorFieldDescr,
     FLAG_POINTER, FLAG_FLOAT)
 from rpython.jit.backend.llsupport.memcpy import memset_fn
-from rpython.jit.backend.llsupport import asmmemmgr
+from rpython.jit.backend.llsupport import asmmemmgr, codemap
 from rpython.rlib.unroll import unrolling_iterable
 
 
@@ -49,7 +49,7 @@
         else:
             self._setup_exception_handling_untranslated()
         self.asmmemmgr = asmmemmgr.AsmMemoryManager()
-        asmmemmgr._memmngr = self.asmmemmgr
+        self.codemap = codemap.CodemapStorage()
         self._setup_frame_realloc(translate_support_code)
         ad = self.gc_ll_descr.getframedescrs(self).arraydescr
         self.signedarraydescr = ad
@@ -79,6 +79,9 @@
     def setup(self):
         pass
 
+    def finish_once(self):
+        self.codemap.finish_once()
+
     def _setup_frame_realloc(self, translate_support_code):
         FUNC_TP = lltype.Ptr(lltype.FuncType([llmemory.GCREF, lltype.Signed],
                                              llmemory.GCREF))
@@ -213,6 +216,7 @@
             for rawstart, rawstop in blocks:
                 self.gc_ll_descr.freeing_block(rawstart, rawstop)
                 self.asmmemmgr.free(rawstart, rawstop)
+                self.codemap.free_asm_block(rawstart, rawstop)
 
     def force(self, addr_of_force_token):
         frame = rffi.cast(jitframe.JITFRAMEPTR, addr_of_force_token)
diff --git a/rpython/jit/backend/llsupport/src/codemap.c b/rpython/jit/backend/llsupport/src/codemap.c
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/llsupport/src/codemap.c
@@ -0,0 +1,204 @@
+#include "src/precommondefs.h"
+
+#ifndef HAS_SKIPLIST
+# error "skiplist.c needs to be included before"
+#endif
+
+volatile int pypy_codemap_currently_invalid = 0;
+
+void pypy_codemap_invalid_set(int value)
+{
+    if (value)
+        __sync_lock_test_and_set(&pypy_codemap_currently_invalid, 1);
+    else
+        __sync_lock_release(&pypy_codemap_currently_invalid);
+}
+
+
+/************************************************************/
+/***  codemap storage                                     ***/
+/************************************************************/
+
+typedef struct {
+    unsigned int machine_code_size;
+    unsigned int bytecode_info_size;
+    long *bytecode_info;
+} codemap_data_t;
+
+static skipnode_t jit_codemap_head;
+
+/*** interface used from codemap.py ***/
+
+RPY_EXTERN
+long pypy_jit_codemap_add(uintptr_t addr, unsigned int machine_code_size,
+                          long *bytecode_info, unsigned int bytecode_info_size)
+{
+    skipnode_t *new = skiplist_malloc(sizeof(codemap_data_t));
+    codemap_data_t *data;
+    if (new == NULL)
+        return -1;   /* too bad */
+
+    new->key = addr;
+    data = (codemap_data_t *)new->data;
+    data->machine_code_size = machine_code_size;
+    data->bytecode_info = bytecode_info;
+    data->bytecode_info_size = bytecode_info_size;
+
+    pypy_codemap_invalid_set(1);
+    skiplist_insert(&jit_codemap_head, new);
+    pypy_codemap_invalid_set(0);
+    return 0;
+}
+
+RPY_EXTERN
+long *pypy_jit_codemap_del(uintptr_t addr)
+{
+    long *result;
+    skipnode_t *node;
+
+    pypy_codemap_invalid_set(1);
+    node = skiplist_remove(&jit_codemap_head, addr);
+    pypy_codemap_invalid_set(0);
+
+    if (node == NULL)
+        return NULL;
+    result = ((codemap_data_t *)node->data)->bytecode_info;
+    free(node);
+    return result;
+}
+
+RPY_EXTERN
+uintptr_t pypy_jit_codemap_firstkey(void)
+{
+    return skiplist_firstkey(&jit_codemap_head);
+}
+
+/*** interface used from pypy/module/_vmprof ***/
+
+RPY_EXTERN
+void *pypy_find_codemap_at_addr(long addr, long* start_addr)
+{
+    skipnode_t *codemap = skiplist_search(&jit_codemap_head, addr);
+    codemap_data_t *data;
+    uintptr_t rel_addr;
+
+    if (codemap == &jit_codemap_head) {
+        if (start_addr)
+            *start_addr = 0;
+        return NULL;
+    }
+
+    rel_addr = (uintptr_t)addr - codemap->key;
+    data = (codemap_data_t *)codemap->data;
+    if (rel_addr >= data->machine_code_size) {
+        if (start_addr)
+            *start_addr = 0;
+        return NULL;
+    }
+
+    if (start_addr)
+        *start_addr = (long)codemap->key;
+    return (void *)codemap;
+}
+
+RPY_EXTERN
+long pypy_yield_codemap_at_addr(void *codemap_raw, long addr,
+                                long *current_pos_addr)
+{
+    // will return consecutive unique_ids from codemap, starting from position
+    // `pos` until addr
+    skipnode_t *codemap = (skipnode_t *)codemap_raw;
+    long current_pos = *current_pos_addr;
+    long rel_addr = addr - codemap->key;
+    long next_start, next_stop;
+    codemap_data_t *data = (codemap_data_t *)codemap->data;
+
+    while (1) {
+        if (current_pos >= data->bytecode_info_size)
+            return 0;
+        next_start = data->bytecode_info[current_pos + 1];
+        if (next_start > rel_addr)
+            return 0;
+        next_stop = data->bytecode_info[current_pos + 2];
+        if (next_stop > rel_addr) {
+            *current_pos_addr = current_pos + 4;
+            return data->bytecode_info[current_pos];
+        }
+        // we need to skip potentially more than one
+        current_pos = data->bytecode_info[current_pos + 3];
+    }
+}
+
+/************************************************************/
+/***  depthmap storage                                    ***/
+/************************************************************/
+
+typedef struct {
+    unsigned int block_size;
+    unsigned int stack_depth;
+} depthmap_data_t;
+
+static skipnode_t jit_depthmap_head;
+
+/*** interface used from codemap.py ***/
+
+RPY_EXTERN
+long pypy_jit_depthmap_add(uintptr_t addr, unsigned int size,
+                           unsigned int stackdepth)
+{
+    skipnode_t *new = skiplist_malloc(sizeof(depthmap_data_t));
+    depthmap_data_t *data;
+    if (new == NULL)
+        return -1;   /* too bad */
+
+    new->key = addr;
+    data = (depthmap_data_t *)new->data;
+    data->block_size = size;
+    data->stack_depth = stackdepth;
+
+    pypy_codemap_invalid_set(1);
+    skiplist_insert(&jit_depthmap_head, new);
+    pypy_codemap_invalid_set(0);
+    return 0;
+}
+
+RPY_EXTERN
+void pypy_jit_depthmap_clear(uintptr_t addr, unsigned int size)
+{
+    uintptr_t search_key = addr + size - 1;
+    if (size == 0)
+        return;
+
+    pypy_codemap_invalid_set(1);
+    while (1) {
+        /* search for all nodes belonging to the range, and remove them */
+        skipnode_t *node = skiplist_search(&jit_depthmap_head, search_key);
+        if (node->key < addr)
+            break;   /* exhausted */
+        skiplist_remove(&jit_depthmap_head, node->key);
+        free(node);
+    }
+    pypy_codemap_invalid_set(0);
+}
+
+/*** interface used from pypy/module/_vmprof ***/
+
+RPY_EXTERN
+long pypy_jit_stack_depth_at_loc(long loc)
+{
+    skipnode_t *depthmap = skiplist_search(&jit_depthmap_head, (uintptr_t)loc);
+    depthmap_data_t *data;
+    uintptr_t rel_addr;
+
+    if (depthmap == &jit_depthmap_head)
+        return -1;
+
+    rel_addr = (uintptr_t)loc - depthmap->key;
+    data = (codemap_data_t *)depthmap->data;
+    if (rel_addr >= data->block_size)
+        return -1;
+
+    return data->stack_depth;
+}
+
+/************************************************************/
diff --git a/rpython/jit/backend/llsupport/src/skiplist.c b/rpython/jit/backend/llsupport/src/skiplist.c
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/llsupport/src/skiplist.c
@@ -0,0 +1,103 @@
+#include 
+#include 
+
+#define HAS_SKIPLIST
+#define SKIPLIST_HEIGHT   8
+
+typedef struct skipnode_s {
+    uintptr_t key;
+    char *data;
+    struct skipnode_s *next[SKIPLIST_HEIGHT];   /* may be smaller */
+} skipnode_t;
+
+static skipnode_t *skiplist_malloc(uintptr_t datasize)
+{
+    char *result;
+    uintptr_t basesize;
+    uintptr_t length = 1;
+    while (length < SKIPLIST_HEIGHT && (rand() & 3) == 0)
+        length++;
+    basesize = sizeof(skipnode_t) -
+               (SKIPLIST_HEIGHT - length) * sizeof(skipnode_t *);
+    result = malloc(basesize + datasize);
+    if (result != NULL) {
+        ((skipnode_t *)result)->data = result + basesize;
+    }
+    return (skipnode_t *)result;
+}
+
+static skipnode_t *skiplist_search(skipnode_t *head, uintptr_t searchkey)
+{
+    /* Returns the skipnode with key closest (but <=) searchkey.
+       Note that if there is no item with key <= searchkey in the list,
+       this will return the head node. */
+    uintptr_t level = SKIPLIST_HEIGHT - 1;
+    while (1) {
+        skipnode_t *next = head->next[level];
+        if (next != NULL && next->key <= searchkey) {
+            head = next;
+        }
+        else {
+            if (level == 0)
+                break;
+            level -= 1;
+        }
+    }
+    return head;
+}
+
+static void skiplist_insert(skipnode_t *head, skipnode_t *new)
+{
+    uintptr_t size0 = sizeof(skipnode_t) -
+                      SKIPLIST_HEIGHT * sizeof(skipnode_t *);
+    uintptr_t height_of_new = (new->data - ((char *)new + size0)) /
+                              sizeof(skipnode_t *);
+
+    uintptr_t level = SKIPLIST_HEIGHT - 1;
+    uintptr_t searchkey = new->key;
+    while (1) {
+        skipnode_t *next = head->next[level];
+        if (next != NULL && next->key <= searchkey) {
+            head = next;
+        }
+        else {
+            if (level < height_of_new) {
+                new->next[level] = next;
+                head->next[level] = new;
+                if (level == 0)
+                    break;
+            }
+            level -= 1;
+        }
+    }
+}
+
+static skipnode_t *skiplist_remove(skipnode_t *head, uintptr_t exact_key)
+{
+    uintptr_t level = SKIPLIST_HEIGHT - 1;
+    while (1) {
+        skipnode_t *next = head->next[level];
+        if (next != NULL && next->key <= exact_key) {
+            if (next->key == exact_key) {
+                head->next[level] = next->next[level];
+                if (level == 0)
+                    return next;    /* successfully removed */
+                level -= 1;
+            }
+            else
+                head = next;
+        }
+        else {
+            if (level == 0)
+                return NULL;    /* 'exact_key' not found! */
+            level -= 1;
+        }
+    }
+}
+
+static uintptr_t skiplist_firstkey(skipnode_t *head)
+{
+    if (head->next[0] == NULL)
+        return 0;
+    return head->next[0]->key;
+}
diff --git a/rpython/jit/backend/llsupport/test/test_asmmemmgr.py b/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
--- a/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
@@ -2,7 +2,7 @@
 from rpython.jit.backend.llsupport.asmmemmgr import AsmMemoryManager
 from rpython.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
 from rpython.jit.backend.llsupport.asmmemmgr import BlockBuilderMixin
-from rpython.jit.backend.llsupport import asmmemmgr
+from rpython.jit.backend.llsupport.codemap import CodemapStorage
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib import debug
 
@@ -96,20 +96,21 @@
 class TestAsmMemoryManager:
 
     def setup_method(self, _):
-        self.memmgr = AsmMemoryManager(min_fragment=8,
+        self.asmmemmgr = AsmMemoryManager(min_fragment=8,
                                        num_indices=10,
                                        large_alloc_size=8192)
+        self.codemap = CodemapStorage()
 
     def teardown_method(self, _):
-        self.memmgr._delete()
+        self.asmmemmgr._delete()
 
     def test_malloc_simple(self):
         for i in range(100):
-            while self.memmgr.total_memory_allocated < 16384:
+            while self.asmmemmgr.total_memory_allocated < 16384:
                 reqsize = random.randrange(1, 200)
-                (start, stop) = self.memmgr.malloc(reqsize, reqsize)
+                (start, stop) = self.asmmemmgr.malloc(reqsize, reqsize)
                 assert reqsize <= stop - start < reqsize + 8
-                assert self.memmgr.total_memory_allocated in [8192, 16384]
+                assert self.asmmemmgr.total_memory_allocated in [8192, 16384]
             self.teardown_method(None)
             self.setup_method(None)
 
@@ -123,7 +124,7 @@
             if got and (random.random() < 0.4 or len(got) == 1000):
                 # free
                 start, stop = got.pop(random.randrange(0, len(got)))
-                self.memmgr.free(start, stop)
+                self.asmmemmgr.free(start, stop)
                 real_use -= (stop - start)
                 assert real_use >= 0
             #
@@ -134,18 +135,18 @@
                     reqmaxsize = reqsize
                 else:
                     reqmaxsize = reqsize + random.randrange(0, 200)
-                (start, stop) = self.memmgr.malloc(reqsize, reqmaxsize)
+                (start, stop) = self.asmmemmgr.malloc(reqsize, reqmaxsize)
                 assert reqsize <= stop - start < reqmaxsize + 8
                 for otherstart, otherstop in got:           # no overlap
                     assert otherstop <= start or stop <= otherstart
                 got.append((start, stop))
                 real_use += (stop - start)
-                if self.memmgr.total_memory_allocated == prev_total:
+                if self.asmmemmgr.total_memory_allocated == prev_total:
                     iterations_without_allocating_more += 1
                     if iterations_without_allocating_more == 40000:
                         break    # ok
                 else:
-                    new_total = self.memmgr.total_memory_allocated
+                    new_total = self.asmmemmgr.total_memory_allocated
                     iterations_without_allocating_more = 0
                     print real_use, new_total
                     # We seem to never see a printed value greater
@@ -172,7 +173,7 @@
         #
         gcrootmap = FakeGcRootMap()
         allblocks = []
-        rawstart = mc.materialize(self.memmgr, allblocks, gcrootmap)
+        rawstart = mc.materialize(self, allblocks, gcrootmap)
         p = rffi.cast(rffi.CArrayPtr(lltype.Char), rawstart)
         assert p[0] == 'X'
         assert p[1] == 'x'
@@ -268,16 +269,3 @@
     md.done()
     assert allblocks == [(1597, 1697), (1797, 1835)]
     assert ops == [('free', 1835, 1897)]
-
-def test_find_jit_frame_depth():
-    mgr = AsmMemoryManager()
-    mgr.register_frame_depth_map(11, [0, 5, 10], [1, 2, 3])
-    mgr.register_frame_depth_map(30, [0, 5, 10], [4, 5, 6])
-    mgr.register_frame_depth_map(0, [0, 5, 10], [7, 8, 9])
-    asmmemmgr._memmngr = mgr
-    assert asmmemmgr.stack_depth_at_loc(13) == 1
-    assert asmmemmgr.stack_depth_at_loc(-3) == -1
-    assert asmmemmgr.stack_depth_at_loc(41) == -1
-    assert asmmemmgr.stack_depth_at_loc(5) == 8
-    assert asmmemmgr.stack_depth_at_loc(17) == 2
-    assert asmmemmgr.stack_depth_at_loc(38) == 5
diff --git a/rpython/jit/backend/llsupport/test/test_codemap.py b/rpython/jit/backend/llsupport/test/test_codemap.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/llsupport/test/test_codemap.py
@@ -0,0 +1,93 @@
+
+from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.jit.backend.llsupport.codemap import stack_depth_at_loc
+from rpython.jit.backend.llsupport.codemap import CodemapStorage, \
+     CodemapBuilder, unpack_traceback, find_codemap_at_addr
+
+NULL = lltype.nullptr(rffi.CArray(lltype.Signed))
+     
+def test_register_codemap():
+    codemap = CodemapStorage()
+    codemap.setup()
+    codemap.register_codemap((100, 20, [13, 14, 15]))
+    codemap.register_codemap((300, 30, [16, 17, 18]))
+    codemap.register_codemap((200, 100, [19, 20, 21, 22, 23]))
+    #
+    raw100 = find_codemap_at_addr(100, NULL)
+    assert find_codemap_at_addr(119, NULL) == raw100
+    assert not find_codemap_at_addr(120, NULL)
+    #
+    raw200 = find_codemap_at_addr(200, NULL)
+    assert raw200 != raw100
+    assert find_codemap_at_addr(299, NULL) == raw200
+    #
+    raw300 = find_codemap_at_addr(329, NULL)
+    assert raw300 != raw100 and raw300 != raw200
+    assert find_codemap_at_addr(300, NULL) == raw300
+    #
+    codemap.free()
+
+def test_find_jit_frame_depth():
+    codemap = CodemapStorage()
+    codemap.setup()
+    codemap.register_frame_depth_map(11, 26, [0, 5, 10], [1, 2, 3])
+    codemap.register_frame_depth_map(30, 41, [0, 5, 10], [4, 5, 6])
+    codemap.register_frame_depth_map(0, 11, [0, 5, 10], [7, 8, 9])
+    assert stack_depth_at_loc(13) == 1
+    assert stack_depth_at_loc(-3) == -1
+    assert stack_depth_at_loc(40) == 6
+    assert stack_depth_at_loc(41) == -1
+    assert stack_depth_at_loc(5) == 8
+    assert stack_depth_at_loc(17) == 2
+    assert stack_depth_at_loc(38) == 5
+    assert stack_depth_at_loc(25) == 3
+    assert stack_depth_at_loc(26) == -1
+    assert stack_depth_at_loc(11) == 1
+    assert stack_depth_at_loc(10) == 9
+    codemap.free_asm_block(11, 26)
+    assert stack_depth_at_loc(11) == -1
+    assert stack_depth_at_loc(13) == -1
+    assert stack_depth_at_loc(-3) == -1
+    assert stack_depth_at_loc(40) == 6
+    assert stack_depth_at_loc(41) == -1
+    assert stack_depth_at_loc(5) == 8
+    assert stack_depth_at_loc(38) == 5
+    assert stack_depth_at_loc(10) == 9
+    codemap.free()
+
+def test_codemaps():
+    builder = CodemapBuilder()
+    builder.debug_merge_point(0, 102, 0)
+    builder.debug_merge_point(0, 102, 13)
+    builder.debug_merge_point(1, 104, 15)
+    builder.debug_merge_point(1, 104, 16)
+    builder.debug_merge_point(2, 106, 20)
+    builder.debug_merge_point(2, 106, 25)
+    builder.debug_merge_point(1, 104, 30)
+    builder.debug_merge_point(0, 102, 35)
+    codemap = CodemapStorage()
+    codemap.setup()
+    codemap.register_codemap(builder.get_final_bytecode(100, 40))
+    builder = CodemapBuilder()
+    builder.debug_merge_point(0, 202, 0)
+    builder.debug_merge_point(0, 202, 10)
+    builder.debug_merge_point(1, 204, 20)
+    builder.debug_merge_point(1, 204, 30)
+    builder.debug_merge_point(2, 206, 40)
+    builder.debug_merge_point(2, 206, 50)
+    builder.debug_merge_point(1, 204, 60)
+    builder.debug_merge_point(0, 202, 70)
+    codemap.register_codemap(builder.get_final_bytecode(200, 100))
+    assert unpack_traceback(110) == [102]
+    assert unpack_traceback(117) == [102, 104]
+    assert unpack_traceback(121) == [102, 104, 106]
+    assert unpack_traceback(131) == [102, 104]
+    assert unpack_traceback(137) == [102]
+    assert unpack_traceback(205) == [202]
+    assert unpack_traceback(225) == [202, 204]
+    assert unpack_traceback(245) == [202, 204, 206]
+    assert unpack_traceback(265) == [202, 204]
+    assert unpack_traceback(275) == [202]
+    codemap.free_asm_block(200, 300)
+    assert unpack_traceback(225) == []

From noreply at buildbot.pypy.org  Fri Apr 17 13:18:41 2015
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 17 Apr 2015 13:18:41 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: introduce a real world example
Message-ID: <20150417111841.BD5DA1C0EDD@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: extradoc
Changeset: r5534:14123b4ae075
Date: 2015-04-17 12:55 +0200
http://bitbucket.org/pypy/extradoc/changeset/14123b4ae075/

Log:	introduce a real world example

diff --git a/talk/pycon-italy-2015/src/decode0.py b/talk/pycon-italy-2015/src/decode0.py
new file mode 100644
--- /dev/null
+++ b/talk/pycon-italy-2015/src/decode0.py
@@ -0,0 +1,25 @@
+import sys
+import struct
+
+P1 = '\x0c\x00\x00\x00"\x00\x00\x00\x07\x00'
+P2 = '\x15\x00\x00\x00+\x00\x00\x00\x08\x00'
+
+PLIST = [P1, P2] * 2000
+
+def read_x(p):
+    return struct.unpack_from('l', p, 0)[0]
+
+def read_y(p):
+    return struct.unpack_from('l', p, 4)[0]
+
+def read_color(p):
+    return struct.unpack_from('i', p, 8)[0]
+
+def main():
+    res = 0
+    for p in PLIST:
+        x = read_x(p)
+        res += x
+    print res
+
+main()
diff --git a/talk/pycon-italy-2015/src/decode1.py b/talk/pycon-italy-2015/src/decode1.py
new file mode 100644
--- /dev/null
+++ b/talk/pycon-italy-2015/src/decode1.py
@@ -0,0 +1,41 @@
+import sys
+import struct
+
+P1 = '\x0c\x00\x00\x00"\x00\x00\x00\x07\x00'
+P2 = '\x15\x00\x00\x00+\x00\x00\x00\x08\x00'
+
+PLIST = [P1, P2] * 2000
+
+class Field(object):
+
+    def __init__(self, fmt, offset):
+        self.fmt = fmt
+        self.offset = offset
+
+
+class Message(object):
+
+    def __init__(self, name, fields):
+        self._name = name
+        self._fields = fields
+
+    def read(self, buf, name):
+        f = self._fields[name]
+        return struct.unpack_from(f.fmt, buf, f.offset)[0]
+
+
+Point = Message('Point', {
+    'x': Field('l', 0),
+    'y': Field('l', 4),
+    'color': Field('i', 8)
+    })
+
+
+def main():
+    res = 0
+    for p in PLIST:
+        x = Point.read(p, 'x')
+        res += x
+    print res
+
+main()
diff --git a/talk/pycon-italy-2015/talk.rst b/talk/pycon-italy-2015/talk.rst
--- a/talk/pycon-italy-2015/talk.rst
+++ b/talk/pycon-italy-2015/talk.rst
@@ -219,6 +219,220 @@
   * inefficient code
 
 
+Virtuals
+--------
+
+- Remove unnecessary allocations
+
+- Remove unnecessary load/store
+
+- "Abstraction for free"
+
+
+Example
+--------
+
+- Real world example
+
+- Decoding binary messages
+
+- Messages: strings of bytes
+
+|small|
+|example<| |small| Point |end_small| |>|
+
+.. sourcecode:: C
+
+    struct Point {
+        long x;
+        long y;
+        int color;
+    }
+
+|end_example|
+|end_small|
+
+
+
+Example: low-level solution
+----------------------------
+
+|scriptsize|
+|example<| |small| decode0.py |end_small| |>|
+
+.. sourcecode:: python
+
+    P1 = '\x0c\x00\x00\x00"\x00\x00\x00\x07\x00\x00\x00'
+    P2 = '\x15\x00\x00\x00+\x00\x00\x00\x08\x00\x00\x00'
+    
+    PLIST = [P1, P2] * 2000
+    
+    def read_x(p):
+        return struct.unpack_from('l', p, 0)[0]
+    
+    def main():
+        res = 0
+        for p in PLIST:
+            x = read_x(p)
+            res += x
+        print res
+
+|end_example|
+|end_scriptsize|
+
+Example: low-level solution
+----------------------------
+
+|scriptsize|
+|example<| |small| decode0.py trace |end_small| |>|
+
+.. sourcecode:: python
+
+    debug_merge_point(1, 1, ' #0 LOAD_GLOBAL')
+    debug_merge_point(1, 1, ' #3 LOOKUP_METHOD')
+    debug_merge_point(1, 1, ' #6 LOAD_CONST')
+    debug_merge_point(1, 1, ' #9 LOAD_FAST')
+    debug_merge_point(1, 1, ' #12 LOAD_CONST')
+    debug_merge_point(1, 1, ' #15 CALL_METHOD')
+    +606: i91 = strlen(p88)
+    +609: i92 = int_lt(i91, 4)
+    guard_false(i92, descr=)
+    +618: i93 = strgetitem(p88, 0)
+    +622: i94 = strgetitem(p88, 1)
+    +632: i95 = int_lshift(i94, 8)
+    +635: i96 = int_or(i93, i95)
+    +637: i97 = strgetitem(p88, 2)
+    +653: i98 = int_lshift(i97, 16)
+    +656: i99 = int_or(i96, i98)
+    +658: i100 = strgetitem(p88, 3)
+    +662: i101 = int_ge(i100, 128)
+    guard_false(i101, descr=)
+    +674: i102 = int_lshift(i100, 24)
+    +677: i103 = int_or(i99, i102)
+    
+|end_example|
+|end_scriptsize|
+
+Example: better API
+---------------------
+
+|scriptsize|
+|example<| |small| decode1.py |end_small| |>|
+
+.. sourcecode:: python
+
+    class Field(object):
+        def __init__(self, fmt, offset):
+            self.fmt = fmt; self.offset = offset
+
+    class Message(object):
+        def __init__(self, name, fields):
+            self._name = name; self._fields = fields
+    
+        def read(self, buf, name):
+            f = self._fields[name]
+            return struct.unpack_from(f.fmt, buf, f.offset)[0]
+
+    Point = Message('Point', {'x': Field('l', 0), 
+                              'y': Field('l', 8),
+                              'color': Field('i', 16)})
+
+    def main():
+        res = 0
+        for p in PLIST:
+            x = Point.read(p, 'x')
+            res += x
+        print res
+
+|end_example|
+|end_scriptsize|
+
+Example: better API
+----------------------------
+
+|scriptsize|
+|example<| |small| decode1.py trace (1) |end_small| |>|
+
+.. sourcecode:: python
+
+    debug_merge_point(1, 1, ' #34 CALL_METHOD')
+    p156 = getfield_gc_pure(p154, descr=)
+    i157 = getfield_gc_pure(p155, descr=)
+    p158 = new_with_vtable(-1228074336)
+    setfield_gc(p158, 0, descr=)
+    call(interpret_trampoline__v238___simple_call__function_i, p158, p156, ...)
+    guard_no_exception(descr=)
+    i159 = getfield_gc(p158, descr=)
+    i160 = int_lt(i157, 0)
+    guard_false(i160, descr=)
+    i161 = strlen(p141)
+    i162 = int_sub(i161, i157)
+    i163 = int_lt(i162, i159)
+    guard_false(i163, descr=)
+    i164 = int_ge(i159, 0)
+    guard_true(i164, descr=)
+    p165 = force_token()
+    p166 = new_with_vtable(-1228077368)
+    p167 = new_with_vtable(-1228077280)
+    p168 = new_with_vtable(-1228267680)
+    setfield_gc(p167, 1, descr=)
+|end_example|
+|end_scriptsize|
+
+Example: better API
+----------------------------
+
+|scriptsize|
+|example<| |small| decode1.py trace (2) |end_small| |>|
+
+.. sourcecode:: python
+
+    p169 = new(descr=)
+    p170 = new_array_clear(0, descr=)
+    p171 = new_with_vtable(-1229823908)
+    setfield_gc(p171, p145, descr=)
+    setfield_gc(p171, p145, descr=)
+    setfield_gc(p171, ConstPtr(null), descr=)
+    setfield_gc(p51, p171, descr=)
+    setfield_gc(p0, p165, descr=)
+    setfield_gc(p168, 1, descr=)
+    setfield_gc(p168, p141, descr=)
+    setfield_gc(p167, p168, descr=)
+    setfield_gc(p167, i157, descr=)
+    setfield_gc(p167, i159, descr=)
+    setfield_gc(p166, p167, descr=)
+    setfield_gc(p166, i159, descr=)
+    setfield_gc(p166, 0, descr=)
+    setfield_gc(p169, 0, descr=)
+    setfield_gc(p169, p170, descr=)
+    setfield_gc(p166, p169, descr=)
+
+|end_example|
+|end_scriptsize|
+
+
+Example: better API
+----------------------------
+
+|scriptsize|
+|example<| |small| decode1.py trace (3) |end_small| |>|
+
+.. sourcecode:: python
+
+    call_may_force(interpret_trampoline__v628___simple_call__function_i), p166, p156, descr=)
+    guard_not_forced(descr=)
+    guard_no_exception(descr=)
+    p172 = getfield_gc(p166, descr=)
+    i173 = getfield_gc(p172, descr=)
+    p174 = new_array_clear(i173, descr=)
+    p175 = getfield_gc(p172, descr=)
+    call(ll_arraycopy__arrayPtr_arrayPtr_Signed_Signed_Signed), p175, p174, 0, 0, i173, descr=)
+    i176 = int_eq(i173, 2)
+    guard_false(i176, descr=)
+
+|end_example|
+|end_scriptsize|
+
 
 
 Contacts, Q&A

From noreply at buildbot.pypy.org  Fri Apr 17 13:18:42 2015
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 17 Apr 2015 13:18:42 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: more slides
Message-ID: <20150417111842.D865C1C0EDD@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: extradoc
Changeset: r5535:60dd04fca106
Date: 2015-04-17 13:10 +0200
http://bitbucket.org/pypy/extradoc/changeset/60dd04fca106/

Log:	more slides

diff --git a/talk/pycon-italy-2015/src/decode2.py b/talk/pycon-italy-2015/src/decode2.py
new file mode 100644
--- /dev/null
+++ b/talk/pycon-italy-2015/src/decode2.py
@@ -0,0 +1,42 @@
+import sys
+import struct
+
+P1 = '\x0c\x00\x00\x00"\x00\x00\x00\x07\x00'
+P2 = '\x15\x00\x00\x00+\x00\x00\x00\x08\x00'
+
+PLIST = [P1, P2] * 2000
+
+class Field(object):
+
+    def __init__(self, fmt, offset):
+        self.fmt = fmt
+        self.offset = offset
+
+def Message(name, fields):
+    class M(object):
+        def read(self, buf, name):
+            f = getattr(self, name)
+            return struct.unpack_from(f.fmt, buf, f.offset)[0]            
+
+    for fname, f in fields.iteritems():
+        setattr(M, fname, f)
+
+    M.__name__ = name
+    return M()
+
+
+Point = Message('Point', {
+    'x': Field('l', 0),
+    'y': Field('l', 4),
+    'color': Field('i', 8)
+    })
+
+
+def main():
+    res = 0
+    for p in PLIST:
+        x = Point.read(p, 'x')
+        res += x
+    print res
+
+main()
diff --git a/talk/pycon-italy-2015/src/decode3.py b/talk/pycon-italy-2015/src/decode3.py
new file mode 100644
--- /dev/null
+++ b/talk/pycon-italy-2015/src/decode3.py
@@ -0,0 +1,32 @@
+import sys
+import struct
+
+P1 = '\x0c\x00\x00\x00"\x00\x00\x00\x07\x00'
+P2 = '\x15\x00\x00\x00+\x00\x00\x00\x08\x00'
+
+PLIST = [P1, P2] * 2000
+
+class Field(object):
+    def __init__(self, fmt, offset):
+        self.fmt = fmt
+        self.offset = offset
+
+    def __get__(self, obj, cls):
+        return struct.unpack_from(self.fmt, obj._buf, self.offset)[0]
+
+class Point(object):
+    def __init__(self, buf):
+        self._buf = buf
+
+    x = Field('l', 0)
+    y = Field('l', 4)
+    color = Field('h', 8)
+
+def main():
+    res = 0
+    for p in PLIST:
+        p = Point(p)
+        res += p.x
+    print res
+
+main()
diff --git a/talk/pycon-italy-2015/talk.rst b/talk/pycon-italy-2015/talk.rst
--- a/talk/pycon-italy-2015/talk.rst
+++ b/talk/pycon-italy-2015/talk.rst
@@ -244,9 +244,9 @@
 .. sourcecode:: C
 
     struct Point {
-        long x;
-        long y;
-        int color;
+        int x;
+        int y;
+        short color;
     }
 
 |end_example|
@@ -433,6 +433,121 @@
 |end_example|
 |end_scriptsize|
 
+Example: faster API
+---------------------
+
+|scriptsize|
+|example<| |small| decode2.py |end_small| |>|
+
+.. sourcecode:: python
+
+    def Message(name, fields):
+        class M(object):
+            def read(self, buf, name):
+                f = getattr(self, name)
+                return struct.unpack_from(f.fmt, buf, f.offset)[0]            
+    
+        for fname, f in fields.iteritems():
+            setattr(M, fname, f)
+    
+        M.__name__ = name
+        return M()
+    
+    Point = Message('Point', {
+        'x': Field('l', 0),
+        'y': Field('l', 4),
+        'color': Field('i', 8)
+        })
+    
+     ...
+     x = Point.read(p, 'x')
+     ...
+
+|end_example|
+|end_scriptsize|
+
+Example: faster API
+----------------------------
+
+|scriptsize|
+|example<| |small| decode2.py trace (3) |end_small| |>|
+
+.. sourcecode:: python
+
+    debug_merge_point(1, 1, ' #36 CALL_METHOD')
+    +670: i104 = strlen(p101)
+    +673: i105 = int_lt(i104, 4)
+    guard_false(i105, descr=)
+    +682: i106 = strgetitem(p101, 0)
+    +686: i107 = strgetitem(p101, 1)
+    +696: i108 = int_lshift(i107, 8)
+    +699: i109 = int_or(i106, i108)
+    +701: i110 = strgetitem(p101, 2)
+    +717: i111 = int_lshift(i110, 16)
+    +720: i112 = int_or(i109, i111)
+    +722: i113 = strgetitem(p101, 3)
+    +726: i114 = int_ge(i113, 128)
+    guard_false(i114, descr=)
+    +738: i115 = int_lshift(i113, 24)
+    +741: i116 = int_or(i112, i115)
+
+|end_example|
+|end_scriptsize|
+
+What happened?
+---------------
+
+- dict lookups inside classes are specialized
+
+- decode1.py
+
+  * ``fields`` is "normal data" and expected to change
+
+  * one JIT code for **all** possible messages
+
+- decode2.py
+
+  * ``fields`` is expected to be constant
+
+  * one JIT code for **each** message
+
+- Behaviour is the same, different performance
+
+
+Example: even better API :)
+-----------------------------
+
+|scriptsize|
+|example<| |small| decode3.py |end_small| |>|
+
+.. sourcecode:: python
+
+    class Field(object):
+        def __init__(self, fmt, offset):
+            self.fmt = fmt
+            self.offset = offset
+    
+        def __get__(self, obj, cls):
+            return struct.unpack_from(self.fmt, obj._buf, self.offset)[0]
+    
+    class Point(object):
+        def __init__(self, buf):
+            self._buf = buf
+    
+        x = Field('l', 0)
+        y = Field('l', 4)
+        color = Field('h', 8)
+    
+    def main():
+        res = 0
+        for p in PLIST:
+            p = Point(p)
+            res += p.x
+        print res
+
+|end_example|
+|end_scriptsize|
+
 
 
 Contacts, Q&A

From noreply at buildbot.pypy.org  Fri Apr 17 13:18:43 2015
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 17 Apr 2015 13:18:43 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: expand virtuals
Message-ID: <20150417111843.E54A11C0EDD@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: extradoc
Changeset: r5536:673c2a4346a1
Date: 2015-04-17 13:13 +0200
http://bitbucket.org/pypy/extradoc/changeset/673c2a4346a1/

Log:	expand virtuals

diff --git a/talk/pycon-italy-2015/talk.rst b/talk/pycon-italy-2015/talk.rst
--- a/talk/pycon-italy-2015/talk.rst
+++ b/talk/pycon-italy-2015/talk.rst
@@ -226,7 +226,18 @@
 
 - Remove unnecessary load/store
 
-- "Abstraction for free"
+|small|
+|example<| |small| virtuals.py |end_small| |>|
+
+.. sourcecode:: python
+
+    res = 0
+    while res < 10000:
+        obj = Foo(x, y, z)
+        res += obj.x
+
+|end_example|
+|end_small|
 
 
 Example

From noreply at buildbot.pypy.org  Fri Apr 17 13:18:45 2015
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 17 Apr 2015 13:18:45 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: add pdf
Message-ID: <20150417111845.0F3B31C0EDD@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni 
Branch: extradoc
Changeset: r5537:0769e28550ef
Date: 2015-04-17 13:19 +0200
http://bitbucket.org/pypy/extradoc/changeset/0769e28550ef/

Log:	add pdf

diff --git a/talk/pycon-italy-2015/talk.pdf b/talk/pycon-italy-2015/talk.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..1492bcaa865783996c4f06a4afc7eabcea54249b
GIT binary patch

[cut]


From noreply at buildbot.pypy.org  Fri Apr 17 15:56:46 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 17 Apr 2015 15:56:46 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof2: in-progress try making enter/leave
 instead of relying on DMP
Message-ID: <20150417135646.31CDD1C1174@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof2
Changeset: r76816:1178d8344723
Date: 2015-04-17 14:16 +0200
http://bitbucket.org/pypy/pypy/changeset/1178d8344723/

Log:	in-progress try making enter/leave instead of relying on DMP

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -1025,6 +1025,12 @@
         else:
             stats.add_merge_point_location(args[1:])
 
+    def execute_enter_portal_frame(self, descr, arg):
+        pass
+
+    def execute_leave_portal_frame(self, descr):
+        pass
+
     def execute_new_with_vtable(self, _, vtable):
         descr = heaptracker.vtable2descr(self.cpu, vtable)
         return self.cpu.bh_new_with_vtable(vtable, descr)
diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -329,6 +329,8 @@
                          rop.ZERO_ARRAY,
                          rop.DEBUG_MERGE_POINT,
                          rop.JIT_DEBUG,
+                         rop.ENTER_PORTAL_FRAME,
+                         rop.LEAVE_PORTAL_FRAME,
                          rop.SETARRAYITEM_RAW,
                          rop.SETINTERIORFIELD_RAW,
                          rop.CALL_RELEASE_GIL,
diff --git a/rpython/jit/metainterp/logger.py b/rpython/jit/metainterp/logger.py
--- a/rpython/jit/metainterp/logger.py
+++ b/rpython/jit/metainterp/logger.py
@@ -134,7 +134,7 @@
     def repr_of_resop(self, op, ops_offset=None):
         if op.getopnum() == rop.DEBUG_MERGE_POINT:
             jd_sd = self.metainterp_sd.jitdrivers_sd[op.getarg(0).getint()]
-            s = jd_sd.warmstate.get_location_str(op.getarglist()[4:])
+            s = jd_sd.warmstate.get_location_str(op.getarglist()[3:])
             s = s.replace(',', '.') # we use comma for argument splitting
             return "debug_merge_point(%d, %d, '%s')" % (op.getarg(1).getint(), op.getarg(2).getint(), s)
         if op.getopnum() == rop.JIT_DEBUG:
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1180,9 +1180,8 @@
         # Note: the logger hides the jd_index argument, so we see in the logs:
         #    debug_merge_point(portal_call_depth, current_call_id, 'location')
         #
-        unique_id = jitdriver_sd.warmstate.get_unique_id(greenkey)
         args = [ConstInt(jd_index), ConstInt(portal_call_depth),
-                ConstInt(current_call_id), ConstInt(unique_id)] + greenkey
+                ConstInt(current_call_id)] + greenkey
         self.metainterp.history.record(rop.DEBUG_MERGE_POINT, args, None)
 
     @arguments("box", "label")
@@ -1772,17 +1771,22 @@
 
     def perform_call(self, jitcode, boxes, greenkey=None):
         # causes the metainterp to enter the given subfunction
-        f = self.newframe(jitcode, greenkey)
+        if greenkey is not None:
+            unique_id = jitcode.jitdriver_sd.warmstate.get_unique_id(greenkey)
+        else:
+            unique_id = -1
+        f = self.newframe(jitcode, greenkey, unique_id)
         f.setup_call(boxes)
         raise ChangeFrame
 
     def is_main_jitcode(self, jitcode):
         return self.jitdriver_sd is not None and jitcode is self.jitdriver_sd.mainjitcode
 
-    def newframe(self, jitcode, greenkey=None):
+    def newframe(self, jitcode, greenkey=None, unique_id=-1):
         if jitcode.jitdriver_sd:
             self.portal_call_depth += 1
             self.call_ids.append(self.current_call_id)
+            self.enter_portal_frame(unique_id)
             self.current_call_id += 1
         if greenkey is not None and self.is_main_jitcode(jitcode):
             self.portal_trace_positions.append(
@@ -1795,11 +1799,20 @@
         self.framestack.append(f)
         return f
 
+    def enter_portal_frame(self, unique_id):
+        self.history.record(rop.ENTER_PORTAL_FRAME,
+                            [ConstInt(unique_id)], None)
+
+    def leave_portal_frame(self):
+        self.history.record(rop.LEAVE_PORTAL_FRAME, [], None)
+
+
     def popframe(self):
         frame = self.framestack.pop()
         jitcode = frame.jitcode
         if jitcode.jitdriver_sd:
             self.portal_call_depth -= 1
+            self.leave_portal_frame()
             self.call_ids.pop()
         if frame.greenkey is not None and self.is_main_jitcode(jitcode):
             self.portal_trace_positions.append(
@@ -2501,7 +2514,10 @@
         # ----- make a new frame -----
         self.portal_call_depth = -1 # always one portal around
         self.framestack = []
-        f = self.newframe(self.jitdriver_sd.mainjitcode)
+        jitdriver_sd = self.jitdriver_sd
+        greenkey = original_boxes[:jitdriver_sd.num_green_args]
+        unique_id = jitdriver_sd.warmstate.get_unique_id(greenkey)
+        f = self.newframe(self.jitdriver_sd.mainjitcode, None, unique_id)
         f.setup_call(original_boxes)
         assert self.portal_call_depth == 0
         self.virtualref_boxes = []
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -522,6 +522,8 @@
     'COND_CALL_GC_WB/1d',       # [objptr] (for the write barrier)
     'COND_CALL_GC_WB_ARRAY/2d', # [objptr, arrayindex] (write barr. for array)
     'DEBUG_MERGE_POINT/*',      # debugging only
+    'ENTER_PORTAL_FRAME/1',     # debugging only
+    'LEAVE_PORTAL_FRAME/0',     # debugging only
     'JIT_DEBUG/*',              # debugging only
     'VIRTUAL_REF_FINISH/2',   # removed before it's passed to the backend
     'COPYSTRCONTENT/5',       # src, dst, srcstart, dststart, length

From noreply at buildbot.pypy.org  Fri Apr 17 15:56:47 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 17 Apr 2015 15:56:47 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof2: improve the tests for enter_portal_frame
Message-ID: <20150417135647.723311C1174@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof2
Changeset: r76817:e521ae5f23f5
Date: 2015-04-17 15:56 +0200
http://bitbucket.org/pypy/pypy/changeset/e521ae5f23f5/

Log:	improve the tests for enter_portal_frame

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -1025,7 +1025,7 @@
         else:
             stats.add_merge_point_location(args[1:])
 
-    def execute_enter_portal_frame(self, descr, arg):
+    def execute_enter_portal_frame(self, descr, *args):
         pass
 
     def execute_leave_portal_frame(self, descr):
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -884,6 +884,8 @@
             insns[opname] = insns.get(opname, 0) + 1
         if expected is not None:
             insns.pop('debug_merge_point', None)
+            insns.pop('enter_portal_frame', None)
+            insns.pop('leave_portal_frame', None)
             assert insns == expected
         for insn, expected_count in check.items():
             getattr(rop, insn.upper())  # fails if 'rop.INSN' does not exist
@@ -901,6 +903,8 @@
     def _check_insns(self, insns, expected, check):
         if expected is not None:
             insns.pop('debug_merge_point', None)
+            insns.pop('enter_portal_frame', None)
+            insns.pop('leave_portal_frame', None)
             insns.pop('label', None)
             assert insns == expected
         for insn, expected_count in check.items():
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1771,22 +1771,23 @@
 
     def perform_call(self, jitcode, boxes, greenkey=None):
         # causes the metainterp to enter the given subfunction
-        if greenkey is not None:
-            unique_id = jitcode.jitdriver_sd.warmstate.get_unique_id(greenkey)
-        else:
-            unique_id = -1
-        f = self.newframe(jitcode, greenkey, unique_id)
+        f = self.newframe(jitcode, greenkey)
         f.setup_call(boxes)
         raise ChangeFrame
 
     def is_main_jitcode(self, jitcode):
         return self.jitdriver_sd is not None and jitcode is self.jitdriver_sd.mainjitcode
 
-    def newframe(self, jitcode, greenkey=None, unique_id=-1):
+    def newframe(self, jitcode, greenkey=None):
         if jitcode.jitdriver_sd:
             self.portal_call_depth += 1
             self.call_ids.append(self.current_call_id)
-            self.enter_portal_frame(unique_id)
+            unique_id = -1
+            if greenkey is not None:
+                unique_id = jitcode.jitdriver_sd.warmstate.get_unique_id(
+                    greenkey)
+                jd_no = jitcode.jitdriver_sd.index
+                self.enter_portal_frame(jd_no, unique_id)
             self.current_call_id += 1
         if greenkey is not None and self.is_main_jitcode(jitcode):
             self.portal_trace_positions.append(
@@ -1799,9 +1800,9 @@
         self.framestack.append(f)
         return f
 
-    def enter_portal_frame(self, unique_id):
+    def enter_portal_frame(self, jd_no, unique_id):
         self.history.record(rop.ENTER_PORTAL_FRAME,
-                            [ConstInt(unique_id)], None)
+                            [ConstInt(jd_no), ConstInt(unique_id)], None)
 
     def leave_portal_frame(self):
         self.history.record(rop.LEAVE_PORTAL_FRAME, [], None)
@@ -2514,10 +2515,7 @@
         # ----- make a new frame -----
         self.portal_call_depth = -1 # always one portal around
         self.framestack = []
-        jitdriver_sd = self.jitdriver_sd
-        greenkey = original_boxes[:jitdriver_sd.num_green_args]
-        unique_id = jitdriver_sd.warmstate.get_unique_id(greenkey)
-        f = self.newframe(self.jitdriver_sd.mainjitcode, None, unique_id)
+        f = self.newframe(self.jitdriver_sd.mainjitcode)
         f.setup_call(original_boxes)
         assert self.portal_call_depth == 0
         self.virtualref_boxes = []
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -522,7 +522,7 @@
     'COND_CALL_GC_WB/1d',       # [objptr] (for the write barrier)
     'COND_CALL_GC_WB_ARRAY/2d', # [objptr, arrayindex] (write barr. for array)
     'DEBUG_MERGE_POINT/*',      # debugging only
-    'ENTER_PORTAL_FRAME/1',     # debugging only
+    'ENTER_PORTAL_FRAME/2',     # debugging only
     'LEAVE_PORTAL_FRAME/0',     # debugging only
     'JIT_DEBUG/*',              # debugging only
     'VIRTUAL_REF_FINISH/2',   # removed before it's passed to the backend
diff --git a/rpython/jit/metainterp/test/test_jitdriver.py b/rpython/jit/metainterp/test/test_jitdriver.py
--- a/rpython/jit/metainterp/test/test_jitdriver.py
+++ b/rpython/jit/metainterp/test/test_jitdriver.py
@@ -195,21 +195,24 @@
         driver = JitDriver(greens=["pc"], reds='auto',
                            get_unique_id=get_unique_id)
 
-        def f():
+        def f(arg):
             i = 0
             pc = 0
-            while i < 10:
+            while i < 30 and pc < 3:
                 driver.jit_merge_point(pc=pc)
                 pc += 1
-                if pc == 3:
+                if arg == 0 and pc == 3:
                     pc = 0
+                if arg == 0:
+                    f(1)
                 i += 1
 
-        self.meta_interp(f, [])
-        loop = get_stats().loops[0]
+        self.meta_interp(f, [0], inline=True)
+        loop = get_stats().loops[1]
         for op in loop.operations:
-            if op.getopname() == 'debug_merge_point':
-                assert op.getarg(3).getint() == op.getarg(4).getint() + 1
-
+            if op.getopname() == 'enter_portal_frame':
+                assert op.getarg(0).getint() == 0
+                assert op.getarg(1).getint() == 1
+                
 class TestLLtype(MultipleJitDriversTests, LLJitMixin):
     pass

From noreply at buildbot.pypy.org  Fri Apr 17 16:31:48 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 17 Apr 2015 16:31:48 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof2: exchange the logic with something much
	simpler
Message-ID: <20150417143148.EC67F1C1388@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof2
Changeset: r76818:680943462021
Date: 2015-04-17 16:31 +0200
http://bitbucket.org/pypy/pypy/changeset/680943462021/

Log:	exchange the logic with something much simpler

diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -575,8 +575,8 @@
             self.mc.BL(self.stack_check_slowpath, c=c.HI)      # call if ip > lr
 
     # cpu interface
-    def assemble_loop(self, logger, loopname, inputargs, operations, looptoken,
-                      log):
+    def assemble_loop(self, jd_id, unique_id, logger, loopname, inputargs,
+                      operations, looptoken, log):
         clt = CompiledLoopToken(self.cpu, looptoken.number)
         looptoken.compiled_loop_token = clt
         clt._debug_nbargs = len(inputargs)
@@ -586,6 +586,9 @@
             assert len(set(inputargs)) == len(inputargs)
 
         self.setup(looptoken)
+        self.codemap_builder.enter_portal_frame(jd_id, unique_id,
+                                        self.mc.get_relative_pos())
+
 
         frame_info = self.datablockwrapper.malloc_aligned(
             jitframe.JITFRAMEINFO_SIZE, alignment=WORD)
diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -373,6 +373,12 @@
         return gcmap
 
     # ------------------------------------------------------------
+    def perform_enter_portal_frame(self, op):
+        self.assembler.enter_portal_frame(op)
+
+    def perform_leave_portal_frame(self, op):
+        self.assembler.leave_portal_frame(op)
+
     def perform_extra(self, op, args, fcond):
         return self.assembler.regalloc_emit_extra(op, args, fcond, self)
 
@@ -392,9 +398,6 @@
         else:
             self.rm._sync_var(v)
 
-    def prepare_op_debug_merge_point(self, op, fcond):
-        self.assembler.codemap.debug_merge_point(op)
-
     def _prepare_op_int_add(self, op, fcond):
         boxes = op.getarglist()
         a0, a1 = boxes
diff --git a/rpython/jit/backend/arm/runner.py b/rpython/jit/backend/arm/runner.py
--- a/rpython/jit/backend/arm/runner.py
+++ b/rpython/jit/backend/arm/runner.py
@@ -56,11 +56,6 @@
     def finish_once(self):
         self.assembler.finish_once()
 
-    def compile_loop(self, inputargs, operations, looptoken,
-                     log=True, name='', logger=None):
-        return self.assembler.assemble_loop(logger, name, inputargs, operations,
-                                            looptoken, log=log)
-
     def compile_bridge(self, faildescr, inputargs, operations,
                        original_loop_token, log=True, logger=None):
         clt = original_loop_token.compiled_loop_token
diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -199,11 +199,15 @@
         guardtok.faildescr.rd_locs = positions
         return fail_descr, target
 
-    def debug_merge_point(self, op):
-        self.codemap_builder.debug_merge_point(op.getarg(1).getint(),
-                                               op.getarg(3).getint(),
+    def enter_portal_frame(self, op):
+        self.codemap_builder.debug_merge_point(op.getarg(0).getint(),
+                                               op.getarg(1).getint(),
                                                self.mc.get_relative_pos())
 
+    def leave_portal_frame(self, op):
+        self.codemap_builder.leave_portal_frame(op.getarg(0).getint(),
+                                                self.mc.get_relative_pos())
+
     def call_assembler(self, op, guard_op, argloc, vloc, result_loc, tmploc):
         self._store_force_index(guard_op)
         descr = op.getdescr()
diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py
--- a/rpython/jit/backend/llsupport/codemap.py
+++ b/rpython/jit/backend/llsupport/codemap.py
@@ -145,29 +145,21 @@
         self.patch_position = []
         self.last_call_depth = -1
 
-    def debug_merge_point(self, call_depth, unique_id, pos):
-        if call_depth != self.last_call_depth:
-            if unique_id == 0: # uninteresting case
-                return
-            assert unique_id & 1 == 0
-            if call_depth > self.last_call_depth:
-                assert call_depth == self.last_call_depth + 1
-                # ^^^ It should never be the case that we see
-                # debug_merge_points that suddenly go more than *one*
-                # call deeper than the previous one (unless we're at
-                # the start of a bridge, handled by
-                # inherit_code_from_position()).
-                self.l.append(unique_id)
-                self.l.append(pos) # <- this is a relative pos
-                self.patch_position.append(len(self.l))
-                self.l.append(0) # marker
-                self.l.append(0) # second marker
-            else:
-                for i in range(self.last_call_depth - call_depth):
-                    to_patch = self.patch_position.pop()
-                    self.l[to_patch] = pos
-                    self.l[to_patch + 1] = len(self.l)
-            self.last_call_depth = call_depth
+    def enter_portal_frame(self, jd_id, unique_id, relpos):
+        if jd_id != 0:
+            return
+        self.l.append(unique_id)
+        self.l.append(relpos)
+        self.patch_position.append(len(self.l))
+        self.l.append(0) # marker
+        self.l.append(0) # second marker
+
+    def leave_portal_frame(self, jd_id, relpos):
+        if jd_id != 0:
+            return
+        to_patch = self.patch_position.pop()
+        self.l[to_patch] = relpos
+        self.l[to_patch + 1] = len(self.l)
 
     def inherit_code_from_position(self, pos):
         lst = unpack_traceback(pos)
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -82,6 +82,12 @@
     def finish_once(self):
         self.codemap.finish_once()
 
+    def compile_loop(self, jd_id, unique_id, inputargs, operations, looptoken,
+                     log=True, name='', logger=None):
+        return self.assembler.assemble_loop(jd_id, unique_id, logger, name,
+                                            inputargs, operations,
+                                            looptoken, log=log)
+
     def _setup_frame_realloc(self, translate_support_code):
         FUNC_TP = lltype.Ptr(lltype.FuncType([llmemory.GCREF, lltype.Signed],
                                              llmemory.GCREF))
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -67,6 +67,8 @@
                 self.emit_pending_zeros()
             elif op.can_malloc():
                 self.emitting_an_operation_that_can_collect()
+            elif op.getopnum() == rop.DEBUG_MERGE_POINT:
+                continue # ignore debug_merge_points
             elif op.getopnum() == rop.LABEL:
                 self.emitting_an_operation_that_can_collect()
                 self.known_lengths.clear()
diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py
--- a/rpython/jit/backend/model.py
+++ b/rpython/jit/backend/model.py
@@ -51,7 +51,7 @@
         """
         return False
 
-    def compile_loop(self, inputargs, operations, looptoken,
+    def compile_loop(self, jd_id, unique_id, inputargs, operations, looptoken,
                      log=True, name='', logger=None):
         """Assemble the given loop.
         Should create and attach a fresh CompiledLoopToken to
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -438,8 +438,8 @@
             self.wb_slowpath[withcards + 2 * withfloats] = rawstart
 
     @rgc.no_release_gil
-    def assemble_loop(self, inputargs, operations, looptoken, log,
-                      loopname, logger):
+    def assemble_loop(self, jd_id, unique_id, logger, loopname, inputargs,
+                      operations, looptoken, log):
         '''adds the following attributes to looptoken:
                _ll_function_addr    (address of the generated func, as an int)
                _ll_loop_code       (debug: addr of the start of the ResOps)
@@ -457,7 +457,8 @@
             assert len(set(inputargs)) == len(inputargs)
 
         self.setup(looptoken)
-
+        self.codemap_builder.enter_portal_frame(jd_id, unique_id,
+                                                self.mc.get_relative_pos())
         frame_info = self.datablockwrapper.malloc_aligned(
             jitframe.JITFRAMEINFO_SIZE, alignment=WORD)
         clt.frame_info = rffi.cast(jitframe.JITFRAMEINFOPTR, frame_info)
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -1323,8 +1323,11 @@
         self.possibly_free_vars_for_op(op)
         assembler.closing_jump(self.jump_target_descr)
 
-    def consider_debug_merge_point(self, op):
-        self.assembler.debug_merge_point(op)
+    def consider_enter_portal_frame(self, op):
+        self.assembler.enter_portal_frame(op)
+
+    def consider_leave_portal_frame(self, op):
+        self.assembler.leave_portal_frame(op)
 
     def consider_jit_debug(self, op):
         pass
diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py
--- a/rpython/jit/backend/x86/runner.py
+++ b/rpython/jit/backend/x86/runner.py
@@ -92,11 +92,6 @@
         lines = machine_code_dump(data, addr, self.backend_name, label_list)
         print ''.join(lines)
 
-    def compile_loop(self, inputargs, operations, looptoken, log=True,
-                     name='', logger=None):
-        return self.assembler.assemble_loop(inputargs, operations, looptoken, log,
-                                            name, logger)
-
     def compile_bridge(self, faildescr, inputargs, operations,
                        original_loop_token, log=True, logger=None):
         clt = original_loop_token.compiled_loop_token
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -303,11 +303,12 @@
             token.original_jitcell_token = trace.original_jitcell_token
 
 
-def do_compile_loop(metainterp_sd, inputargs, operations, looptoken,
-                    log=True, name=''):
+def do_compile_loop(jd_id, unique_id, metainterp_sd, inputargs, operations,
+                    looptoken, log=True, name=''):
     metainterp_sd.logger_ops.log_loop(inputargs, operations, -2,
                                       'compiling', name=name)
-    return metainterp_sd.cpu.compile_loop(inputargs, operations, looptoken,
+    return metainterp_sd.cpu.compile_loop(jd_id, unique_id, inputargs,
+                                          operations, looptoken,
                                           log=log, name=name,
                                           logger=metainterp_sd.logger_ops)
 
@@ -347,7 +348,9 @@
     debug_start("jit-backend")
     try:
         loopname = jitdriver_sd.warmstate.get_location_str(greenkey)
-        asminfo = do_compile_loop(metainterp_sd, loop.inputargs,
+        unique_id = jitdriver_sd.warmstate.get_unique_id(greenkey)
+        asminfo = do_compile_loop(jitdriver_sd.index, unique_id, metainterp_sd,
+                                  loop.inputargs,
                                   operations, original_jitcell_token,
                                   name=loopname)
     finally:
@@ -925,7 +928,7 @@
     ]
     operations[1].setfailargs([])
     operations = get_deep_immutable_oplist(operations)
-    cpu.compile_loop(inputargs, operations, jitcell_token, log=False)
+    cpu.compile_loop(0, -1, inputargs, operations, jitcell_token, log=False)
     if memory_manager is not None:    # for tests
         memory_manager.keep_loop_alive(jitcell_token)
     return jitcell_token
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1804,8 +1804,8 @@
         self.history.record(rop.ENTER_PORTAL_FRAME,
                             [ConstInt(jd_no), ConstInt(unique_id)], None)
 
-    def leave_portal_frame(self):
-        self.history.record(rop.LEAVE_PORTAL_FRAME, [], None)
+    def leave_portal_frame(self, jd_no):
+        self.history.record(rop.LEAVE_PORTAL_FRAME, [ConstInt(jd_no)], None)
 
 
     def popframe(self):
@@ -1813,7 +1813,7 @@
         jitcode = frame.jitcode
         if jitcode.jitdriver_sd:
             self.portal_call_depth -= 1
-            self.leave_portal_frame()
+            self.leave_portal_frame(jitcode.jitdriver_sd.index)
             self.call_ids.pop()
         if frame.greenkey is not None and self.is_main_jitcode(jitcode):
             self.portal_trace_positions.append(
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -523,7 +523,7 @@
     'COND_CALL_GC_WB_ARRAY/2d', # [objptr, arrayindex] (write barr. for array)
     'DEBUG_MERGE_POINT/*',      # debugging only
     'ENTER_PORTAL_FRAME/2',     # debugging only
-    'LEAVE_PORTAL_FRAME/0',     # debugging only
+    'LEAVE_PORTAL_FRAME/1',     # debugging only
     'JIT_DEBUG/*',              # debugging only
     'VIRTUAL_REF_FINISH/2',   # removed before it's passed to the backend
     'COPYSTRCONTENT/5',       # src, dst, srcstart, dststart, length

From noreply at buildbot.pypy.org  Fri Apr 17 16:41:52 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 17 Apr 2015 16:41:52 +0200 (CEST)
Subject: [pypy-commit] pypy object-dtype2: add failing test (passes with -A)
Message-ID: <20150417144152.EF9EF1C051C@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: object-dtype2
Changeset: r76819:b72789c9b72e
Date: 2015-04-17 17:41 +0300
http://bitbucket.org/pypy/pypy/changeset/b72789c9b72e/

Log:	add failing test (passes with -A)

diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -6,7 +6,7 @@
 from pypy.interpreter.typedef import (TypeDef, GetSetProperty,
                                       interp_attrproperty, interp_attrproperty_w)
 from rpython.rlib import jit
-from rpython.rlib.objectmodel import specialize, compute_hash
+from rpython.rlib.objectmodel import specialize, compute_hash, we_are_translated
 from rpython.rlib.rarithmetic import r_longlong, r_ulonglong
 from pypy.module.micronumpy import types, boxes, base, support, constants as NPY
 from pypy.module.micronumpy.appbridge import get_appbridge_cache
@@ -496,8 +496,17 @@
 
 
 def dtype_from_spec(space, w_spec):
-    w_lst = get_appbridge_cache(space).call_method(space,
-        'numpy.core._internal', '_commastring', Arguments(space, [w_spec]))
+
+    if we_are_translated():
+        w_lst = get_appbridge_cache(space).call_method(space,
+            'numpy.core._internal', '_commastring', Arguments(space, [w_spec]))
+    else:
+        # testing, handle manually
+        if space.eq_w(w_spec, space.wrap('u4,u4,u4')):
+            w_lst = space.newlist([space.wrap('u4')]*3)
+        else:
+            raise oefmt(space.w_RuntimeError,
+                    "cannot parse w_spec")
     if not space.isinstance_w(w_lst, space.w_list) or space.len_w(w_lst) < 1:
         raise oefmt(space.w_RuntimeError,
                     "_commastring is not returning a list with len >= 1")
diff --git a/pypy/module/micronumpy/test/test_object_arrays.py b/pypy/module/micronumpy/test/test_object_arrays.py
--- a/pypy/module/micronumpy/test/test_object_arrays.py
+++ b/pypy/module/micronumpy/test/test_object_arrays.py
@@ -92,3 +92,17 @@
         del fiveOs
         gc.collect()
         assert a[2].whatami() == 'an object'
+
+    def test_array_interface(self):
+        import numpy as np
+        class DummyArray(object):
+            def __init__(self, interface, base=None):
+                self.__array_interface__ = interface
+                self.base = base
+        a = np.array([(1, 2, 3)], dtype='u4,u4,u4')
+        b = np.array([(1, 2, 3), (4, 5, 6), (7, 8, 9)], dtype='u4,u4,u4')
+        interface = dict(a.__array_interface__)
+        interface['shape'] = tuple([3])
+        interface['strides'] = tuple([0])
+        c = np.array(DummyArray(interface, base=a))
+        assert (c == np.array([(1, 2, 3), (1, 2, 3), (1, 2, 3)], dtype='u4,u4,u4') ).all()

From noreply at buildbot.pypy.org  Fri Apr 17 16:46:12 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 17 Apr 2015 16:46:12 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Needs another level of type caching,
 to ensure that there "is" only one
Message-ID: <20150417144612.8E0F51C051C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1739:2c825ace4981
Date: 2015-04-17 16:46 +0200
http://bitbucket.org/cffi/cffi/changeset/2c825ace4981/

Log:	Needs another level of type caching, to ensure that there "is" only
	one of each type around. The _cffi_backend code depends on that.

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -132,6 +132,7 @@
 #define CT_WITH_VAR_ARRAY     1048576
 #define CT_IS_UNSIZED_CHAR_A  2097152
 #define CT_LAZY_FIELD_LIST    4194304
+#define CT_USES_LOCAL         8388608
 #define CT_PRIMITIVE_ANY  (CT_PRIMITIVE_SIGNED |        \
                            CT_PRIMITIVE_UNSIGNED |      \
                            CT_PRIMITIVE_CHAR |          \
@@ -3544,7 +3545,7 @@
 
     td->ct_size = sizeof(void *);
     td->ct_length = -1;
-    td->ct_flags = CT_POINTER;
+    td->ct_flags = CT_POINTER | (ctitem->ct_flags & CT_USES_LOCAL);
     if (ctitem->ct_flags & (CT_STRUCT|CT_UNION))
         td->ct_flags |= CT_IS_PTR_TO_OWNED;
     if (ctitem->ct_flags & CT_VOID)
@@ -3633,7 +3634,7 @@
     td->ct_stuff = (PyObject *)ctptr;
     td->ct_size = arraysize;
     td->ct_length = length;
-    td->ct_flags = flags;
+    td->ct_flags = flags | (ctptr->ct_flags & CT_USES_LOCAL);
     return (PyObject *)td;
 }
 
@@ -3665,7 +3666,7 @@
 
     td->ct_size = -1;
     td->ct_length = -1;
-    td->ct_flags = flag | CT_IS_OPAQUE;
+    td->ct_flags = flag | CT_IS_OPAQUE | CT_USES_LOCAL;
     td->ct_extra = NULL;
     memcpy(td->ct_name, name, namelen + 1);
     td->ct_name_position = namelen;
@@ -4348,6 +4349,8 @@
                                           int ellipsis)
 {
     CTypeDescrObject *fct;
+    Py_ssize_t i, nargs;
+    int all_flags;
 
     fb->nb_bytes = 0;
     fb->bufferp = NULL;
@@ -4369,9 +4372,16 @@
         goto error;
     assert(fb->bufferp == fct->ct_name + fb->nb_bytes);
 
+    all_flags = fresult->ct_flags;
+    nargs = PyTuple_GET_SIZE(fargs);
+    for (i = 0; i < nargs; i++) {
+        CTypeDescrObject *farg = (CTypeDescrObject *)PyTuple_GET_ITEM(fargs, i);
+        all_flags |= farg->ct_flags;
+    }
+
     fct->ct_extra = NULL;
     fct->ct_size = sizeof(void(*)(void));
-    fct->ct_flags = CT_FUNCTIONPTR;
+    fct->ct_flags = CT_FUNCTIONPTR | (all_flags & CT_USES_LOCAL);
     return fct;
 
  error:
@@ -4484,8 +4494,6 @@
         Py_INCREF(o);
         PyTuple_SET_ITEM(fct->ct_stuff, 2 + i, o);
     }
-    fct->ct_size = sizeof(void(*)(void));
-    fct->ct_flags = CT_FUNCTIONPTR;
     return (PyObject *)fct;
 
  error:
@@ -4825,7 +4833,7 @@
     td->ct_size = basetd->ct_size;
     td->ct_length = basetd->ct_length;   /* alignment */
     td->ct_extra = basetd->ct_extra;     /* ffi type  */
-    td->ct_flags = basetd->ct_flags | CT_IS_ENUM;
+    td->ct_flags = basetd->ct_flags | CT_IS_ENUM | CT_USES_LOCAL;
     td->ct_name_position = name_size - 1;
     return (PyObject *)td;
 
diff --git a/new/cffi1_module.c b/new/cffi1_module.c
--- a/new/cffi1_module.c
+++ b/new/cffi1_module.c
@@ -55,7 +55,7 @@
     if (ffi == NULL || PyModule_AddObject(m, "ffi", (PyObject *)ffi) < 0)
         return -1;
 
-    LibObject *lib = lib_internal_new(ctx, module_name);
+    LibObject *lib = lib_internal_new(ffi->types_builder, module_name);
     if (lib == NULL || PyModule_AddObject(m, "lib", (PyObject *)lib) < 0)
         return -1;
 
diff --git a/new/ffi_obj.c b/new/ffi_obj.c
--- a/new/ffi_obj.c
+++ b/new/ffi_obj.c
@@ -20,10 +20,10 @@
 
 struct FFIObject_s {
     PyObject_HEAD
-    PyObject *types_dict;
     PyObject *gc_wrefs;
     struct _cffi_parse_info_s info;
     int ctx_is_static;
+    builder_c_t *types_builder;
     _cffi_opcode_t internal_output[FFI_COMPLEXITY_OUTPUT];
 };
 
@@ -31,10 +31,6 @@
                                    const struct _cffi_type_context_s *ctx,
                                    int ctx_is_static)
 {
-    PyObject *dict = PyDict_New();
-    if (dict == NULL)
-        return NULL;
-
     FFIObject *ffi;
     if (ctx_is_static) {
         ffi = (FFIObject *)PyObject_GC_New(FFIObject, ffitype);
@@ -44,11 +40,14 @@
     else {
         ffi = (FFIObject *)ffitype->tp_alloc(ffitype, 0);
     }
-    if (ffi == NULL) {
-        Py_DECREF(dict);
+    if (ffi == NULL)
+        return NULL;
+
+    ffi->types_builder = new_builder_c(ctx);
+    if (ffi->types_builder == NULL) {
+        Py_DECREF(ffi);
         return NULL;
     }
-    ffi->types_dict = dict;
     ffi->gc_wrefs = NULL;
     ffi->info.ctx = ctx;
     ffi->info.output = ffi->internal_output;
@@ -60,29 +59,17 @@
 static void ffi_dealloc(FFIObject *ffi)
 {
     PyObject_GC_UnTrack(ffi);
-    Py_DECREF(ffi->types_dict);
     Py_XDECREF(ffi->gc_wrefs);
 
-    if (!ffi->ctx_is_static) {
-        const void *mem[] = {ffi->info.ctx->types,
-                             ffi->info.ctx->globals,
-                             ffi->info.ctx->struct_unions,
-                             ffi->info.ctx->fields,
-                             ffi->info.ctx->enums,
-                             ffi->info.ctx->typenames,
-                             ffi->info.ctx};
-        int i;
-        for (i = 0; i < sizeof(mem) / sizeof(*mem); i++) {
-            if (mem[i] != NULL)
-                PyMem_Free((void *)mem[i]);
-        }
-    }
+    if (!ffi->ctx_is_static)
+        free_builder_c(ffi->types_builder);
+
     Py_TYPE(ffi)->tp_free((PyObject *)ffi);
 }
 
 static int ffi_traverse(FFIObject *ffi, visitproc visit, void *arg)
 {
-    Py_VISIT(ffi->types_dict);
+    Py_VISIT(ffi->types_builder->types_dict);
     Py_VISIT(ffi->gc_wrefs);
     return 0;
 }
@@ -128,9 +115,12 @@
        Does not return a new reference!
     */
     if ((accept & ACCEPT_STRING) && PyText_Check(arg)) {
-        PyObject *x = PyDict_GetItem(ffi->types_dict, arg);
-        if (x != NULL && CTypeDescr_Check(x))
+        PyObject *types_dict = ffi->types_builder->types_dict;
+        PyObject *x = PyDict_GetItem(types_dict, arg);
+        if (x != NULL) {
+            assert(CTypeDescr_Check(x));
             return (CTypeDescrObject *)x;
+        }
 
         char *input_text = PyText_AS_UTF8(arg);
         int index = parse_c_type(&ffi->info, input_text);
@@ -143,23 +133,20 @@
                          input_text, spaces);
             return NULL;
         }
-        CTypeDescrObject *ct = realize_c_type(ffi->info.ctx,
+        CTypeDescrObject *ct = realize_c_type(ffi->types_builder,
                                               ffi->info.output, index);
-        if (ct == NULL)
-            return NULL;
-
-        char *normalized_text = ct->ct_name;
-        x = PyDict_GetItemString(ffi->types_dict, normalized_text);
-        if (x == NULL) {
-            PyDict_SetItemString(ffi->types_dict, normalized_text,
-                                 (PyObject *)ct);
+        if (ct != NULL) {
+            /* Cache under the name given by 'arg', in addition to the
+               fact that the same ct is probably already cached under
+               its standardized name.  In a few cases, it is not, e.g.
+               if it is a primitive; for the purpose of this function,
+               the important point is the following line, which makes
+               sure that in any case the next _ffi_type() with the same
+               'arg' will succeed early, in PyDict_GetItem() above.
+            */
+            if (PyDict_SetItem(types_dict, arg, (PyObject *)ct) < 0)
+                return NULL;
         }
-        else {
-            Py_INCREF(x);
-            Py_DECREF(ct);
-            ct = (CTypeDescrObject *)x;
-        }
-        PyDict_SetItem(ffi->types_dict, arg, (PyObject *)ct);
         return ct;
     }
     else if ((accept & ACCEPT_CTYPE) && CTypeDescr_Check(arg)) {
diff --git a/new/lib_obj.c b/new/lib_obj.c
--- a/new/lib_obj.c
+++ b/new/lib_obj.c
@@ -13,13 +13,13 @@
 
 struct CPyExtFunc_s {
     PyMethodDef md;
-    const struct _cffi_type_context_s *ctx;
+    builder_c_t *types_builder;
     int type_index;
 };
 
 struct LibObject_s {
     PyObject_HEAD
-    const struct _cffi_type_context_s *l_ctx;  /* ctx object */
+    builder_c_t *l_types_builder; /* same as the one on the ffi object */
     PyObject *l_dict;           /* content, built lazily */
     PyObject *l_libname;        /* some string that gives the name of the lib */
 };
@@ -59,7 +59,7 @@
     if (xfunc->md.ml_name == NULL)
         goto no_memory;
 
-    xfunc->ctx = lib->l_ctx;
+    xfunc->types_builder = lib->l_types_builder;
     xfunc->type_index = _CFFI_GETARG(g->type_op);
 
     return PyCFunction_NewEx(&xfunc->md, NULL, lib->l_libname);
@@ -73,17 +73,11 @@
 {
     /* does not return a new reference! */
 
-    if (lib->l_ctx == NULL) {
-        PyErr_Format(FFIError, "lib '%.200s' is already closed",
-                     PyText_AS_UTF8(lib->l_libname));
-        return NULL;
-    }
-
     char *s = PyText_AsUTF8(name);
     if (s == NULL)
         return NULL;
 
-    int index = search_in_globals(lib->l_ctx, s, strlen(s));
+    int index = search_in_globals(&lib->l_types_builder->ctx, s, strlen(s));
     if (index < 0) {
         PyErr_Format(PyExc_AttributeError,
                      "lib '%.200s' has no function,"
@@ -93,7 +87,7 @@
         return NULL;
     }
 
-    const struct _cffi_global_s *g = &lib->l_ctx->globals[index];
+    const struct _cffi_global_s *g = &lib->l_types_builder->ctx.globals[index];
     PyObject *x;
     CTypeDescrObject *ct;
 
@@ -136,7 +130,8 @@
     {
         /* a constant which is not of integer type */
         char *data;
-        ct = realize_c_type(lib->l_ctx, lib->l_ctx->types,
+        ct = realize_c_type(lib->l_types_builder,
+                            lib->l_types_builder->ctx.types,
                             _CFFI_GETARG(g->type_op));
         if (ct == NULL)
             return NULL;
@@ -151,7 +146,8 @@
 
     case _CFFI_OP_GLOBAL_VAR:
         /* global variable of the exact type specified here */
-        ct = realize_c_type(lib->l_ctx, lib->l_ctx->types,
+        ct = realize_c_type(lib->l_types_builder,
+                            lib->l_types_builder->ctx.types,
                             _CFFI_GETARG(g->type_op));
         if (ct == NULL)
             return NULL;
@@ -215,8 +211,8 @@
 
 static PyObject *lib_dir(LibObject *lib, PyObject *noarg)
 {
-    const struct _cffi_global_s *g = lib->l_ctx->globals;
-    int total = lib->l_ctx->num_globals;
+    const struct _cffi_global_s *g = lib->l_types_builder->ctx.globals;
+    int total = lib->l_types_builder->ctx.num_globals;
 
     PyObject *lst = PyList_New(total);
     if (lst == NULL)
@@ -277,7 +273,7 @@
     offsetof(LibObject, l_dict),                /* tp_dictoffset */
 };
 
-static LibObject *lib_internal_new(const struct _cffi_type_context_s *ctx,
+static LibObject *lib_internal_new(builder_c_t *types_builder,
                                    char *module_name)
 {
     LibObject *lib;
@@ -295,7 +291,7 @@
     if (lib == NULL)
         return NULL;
 
-    lib->l_ctx = ctx;
+    lib->l_types_builder = types_builder;
     lib->l_dict = dict;
     lib->l_libname = libname;
     return lib;
diff --git a/new/parse_c_type.h b/new/parse_c_type.h
--- a/new/parse_c_type.h
+++ b/new/parse_c_type.h
@@ -71,7 +71,7 @@
     int type_index;          // -> _cffi_types, on a OP_STRUCT_UNION
     int flags;               // CT_UNION?  CT_IS_OPAQUE?
     size_t size;
-    size_t alignment;
+    int alignment;
     int first_field_index;   // -> _cffi_fields array
     int num_fields;
 };
diff --git a/new/realize_c_type.c b/new/realize_c_type.c
--- a/new/realize_c_type.c
+++ b/new/realize_c_type.c
@@ -1,8 +1,118 @@
+
+typedef struct {
+    struct _cffi_type_context_s ctx;   /* inlined substructure */
+    PyObject *types_dict;
+} builder_c_t;
+
 
 static PyObject *all_primitives[_CFFI__NUM_PRIM];
 
+static PyObject *fetch_global_types_dict(void)
+{
+    static PyObject *result = NULL;
+    if (!result)
+        result = PyDict_New();
+    return result;
+}
 
-PyObject *build_primitive_type(int num)
+static void free_builder_c(builder_c_t *builder)
+{
+    Py_XDECREF(builder->types_dict);
+
+    const void *mem[] = {builder->ctx.types,
+                         builder->ctx.globals,
+                         builder->ctx.struct_unions,
+                         builder->ctx.fields,
+                         builder->ctx.enums,
+                         builder->ctx.typenames};
+    int i;
+    for (i = 0; i < sizeof(mem) / sizeof(*mem); i++) {
+        if (mem[i] != NULL)
+            PyMem_Free((void *)mem[i]);
+    }
+    PyMem_Free(builder);
+}
+
+static builder_c_t *new_builder_c(const struct _cffi_type_context_s *ctx)
+{
+    PyObject *ldict = PyDict_New();
+    if (ldict == NULL)
+        return NULL;
+
+    builder_c_t *builder = PyMem_Malloc(sizeof(builder_c_t));
+    if (builder == NULL) {
+        Py_DECREF(ldict);
+        PyErr_NoMemory();
+        return NULL;
+    }
+    builder->ctx = *ctx;
+    builder->types_dict = ldict;
+    return builder;
+}
+
+static PyObject *get_unique_type(builder_c_t *builder, PyObject *x)
+{
+    /* Replace the CTypeDescrObject 'x' with a standardized one.
+       This either just returns x, or x is decrefed and a new reference
+       to the standard equivalent is returned.
+
+       In this function, 'x' always contains a reference that must be
+       decrefed, and 'y' never does.
+    */
+    CTypeDescrObject *ct = (CTypeDescrObject *)x;
+
+    /* XXX maybe change the type of ct_name to be a real 'PyObject *'? */
+    PyObject *name = PyString_FromString(ct->ct_name);
+    if (name == NULL)
+        goto no_memory;
+
+    PyObject *y = PyDict_GetItem(builder->types_dict, name);
+    if (y != NULL) {
+        /* Already found the same ct_name in the dict.  Return the old one. */
+        Py_INCREF(y);
+        Py_DECREF(x);
+        x = y;
+        goto done;
+    }
+
+    if (!(ct->ct_flags & CT_USES_LOCAL)) {
+        /* The type is not "local", i.e. does not make use of any struct,
+           union or enum.  This means it should be shared across independent
+           ffi instances.  Look it up and possibly add it to the global
+           types dict.
+        */
+        PyObject *gdict = fetch_global_types_dict();
+        if (gdict == NULL)
+            goto no_memory;
+
+        y = PyDict_GetItem(gdict, name);
+        if (y != NULL) {
+            Py_INCREF(y);
+            Py_DECREF(x);
+            x = y;
+        }
+        else {
+            /* Not found in the global dictionary.  Put it there. */
+            if (PyDict_SetItem(gdict, name, x) < 0)
+                goto no_memory;
+        }
+    }
+
+    /* Set x in the local dict. */
+    if (PyDict_SetItem(builder->types_dict, name, x) < 0)
+        goto no_memory;
+
+ done:
+    Py_DECREF(name);
+    return x;
+
+ no_memory:
+    Py_XDECREF(name);
+    Py_DECREF(x);
+    return NULL;
+}
+
+static PyObject *build_primitive_type(int num)
 {
     /* XXX too many translations between here and new_primitive_type() */
     static const char *primitive_name[] = {
@@ -58,7 +168,7 @@
 
 
 static PyObject *
-_realize_c_type_or_func(const struct _cffi_type_context_s *ctx,
+_realize_c_type_or_func(builder_c_t *builder,
                         _cffi_opcode_t opcodes[], int index);  /* forward */
 
 
@@ -67,10 +177,9 @@
    reference.
 */
 static CTypeDescrObject *
-realize_c_type(const struct _cffi_type_context_s *ctx,
-               _cffi_opcode_t opcodes[], int index)
+realize_c_type(builder_c_t *builder, _cffi_opcode_t opcodes[], int index)
 {
-    PyObject *x = _realize_c_type_or_func(ctx, opcodes, index);
+    PyObject *x = _realize_c_type_or_func(builder, opcodes, index);
     if (x == NULL || CTypeDescr_Check(x)) {
         return (CTypeDescrObject *)x;
     }
@@ -91,7 +200,7 @@
 }
 
 static PyObject *
-_realize_c_type_or_func(const struct _cffi_type_context_s *ctx,
+_realize_c_type_or_func(builder_c_t *builder,
                         _cffi_opcode_t opcodes[], int index)
 {
     PyObject *x, *y, *z;
@@ -114,11 +223,12 @@
         break;
 
     case _CFFI_OP_POINTER:
-        y = _realize_c_type_or_func(ctx, opcodes, _CFFI_GETARG(op));
+        y = _realize_c_type_or_func(builder, opcodes, _CFFI_GETARG(op));
         if (y == NULL)
             return NULL;
         if (CTypeDescr_Check(y)) {
             x = new_pointer_type((CTypeDescrObject *)y);
+            x = get_unique_type(builder, x);
         }
         else {
             assert(PyTuple_Check(y));   /* from _CFFI_OP_FUNCTION */
@@ -132,14 +242,16 @@
         length = (Py_ssize_t)opcodes[index + 1];
         /* fall-through */
     case _CFFI_OP_OPEN_ARRAY:
-        y = (PyObject *)realize_c_type(ctx, opcodes, _CFFI_GETARG(op));
+        y = (PyObject *)realize_c_type(builder, opcodes, _CFFI_GETARG(op));
         if (y == NULL)
             return NULL;
         z = new_pointer_type((CTypeDescrObject *)y);
+        z = get_unique_type(builder, z);
         Py_DECREF(y);
         if (z == NULL)
             return NULL;
         x = new_array_type((CTypeDescrObject *)z, length);
+        x = get_unique_type(builder, x);
         Py_DECREF(z);
         break;
 
@@ -148,8 +260,8 @@
         const struct _cffi_struct_union_s *s;
         _cffi_opcode_t op2;
 
-        s = &ctx->struct_unions[_CFFI_GETARG(op)];
-        op2 = ctx->types[s->type_index];
+        s = &builder->ctx.struct_unions[_CFFI_GETARG(op)];
+        op2 = builder->ctx.types[s->type_index];
         if ((((uintptr_t)op2) & 1) == 0) {
             x = (PyObject *)op2;
             Py_INCREF(x);
@@ -174,7 +286,7 @@
                 ct->ct_length = s->alignment;
                 ct->ct_flags &= ~CT_IS_OPAQUE;
                 ct->ct_flags |= CT_LAZY_FIELD_LIST;
-                ct->ct_extra = (void *)ctx;
+                ct->ct_extra = builder;
             }
 
             /* We are going to update the "primary" OP_STRUCT_OR_UNION
@@ -184,7 +296,7 @@
                next time we walk the same current slot, we'll find the
                'x' object in the primary slot (op2, above) and then we
                will update the current slot. */
-            opcodes = ctx->types;
+            opcodes = builder->ctx.types;
             index = s->type_index;
         }
         break;
@@ -195,7 +307,7 @@
         PyObject *fargs;
         int i, base_index, num_args;
 
-        y = (PyObject *)realize_c_type(ctx, opcodes, _CFFI_GETARG(op));
+        y = (PyObject *)realize_c_type(builder, opcodes, _CFFI_GETARG(op));
         if (y == NULL)
             return NULL;
 
@@ -212,7 +324,7 @@
         }
 
         for (i = 0; i < num_args; i++) {
-            z = (PyObject *)realize_c_type(ctx, opcodes, base_index + i);
+            z = (PyObject *)realize_c_type(builder, opcodes, base_index + i);
             if (z == NULL) {
                 Py_DECREF(fargs);
                 Py_DECREF(y);
@@ -222,6 +334,7 @@
         }
 
         z = new_function_type(fargs, (CTypeDescrObject *)y, 0, FFI_DEFAULT_ABI);
+        z = get_unique_type(builder, z);
         Py_DECREF(fargs);
         Py_DECREF(y);
         if (z == NULL)
@@ -234,7 +347,7 @@
     }
 
     case _CFFI_OP_NOOP:
-        x = _realize_c_type_or_func(ctx, opcodes, _CFFI_GETARG(op));
+        x = _realize_c_type_or_func(builder, opcodes, _CFFI_GETARG(op));
         break;
 
     case _CFFI_OP_TYPENAME:
@@ -242,8 +355,8 @@
         /* essential: the TYPENAME opcode resolves the type index looked
            up in the 'ctx->typenames' array, but it does so in 'ctx->types'
            instead of in 'opcodes'! */
-        int type_index = ctx->typenames[_CFFI_GETARG(op)].type_index;
-        x = _realize_c_type_or_func(ctx, ctx->types, type_index);
+        int type_index = builder->ctx.typenames[_CFFI_GETARG(op)].type_index;
+        x = _realize_c_type_or_func(builder, builder->ctx.types, type_index);
         break;
     }
 
@@ -252,7 +365,7 @@
         return NULL;
     }
 
-    if (x != NULL && opcodes == ctx->types) {
+    if (x != NULL && opcodes == builder->ctx.types) {
         assert((((uintptr_t)x) & 1) == 0);
         assert((((uintptr_t)opcodes[index]) & 1) == 1);
         Py_INCREF(x);
@@ -268,8 +381,8 @@
     if (ct->ct_flags & CT_LAZY_FIELD_LIST) {
         assert(!(ct->ct_flags & CT_IS_OPAQUE));
 
-        const struct _cffi_type_context_s *ctx = ct->ct_extra;
-        assert(ctx != NULL);
+        builder_c_t *builder = ct->ct_extra;
+        assert(builder != NULL);
 
         char *p = ct->ct_name;
         if (memcmp(p, "struct ", 7) == 0)
@@ -277,12 +390,13 @@
         else if (memcmp(p, "union ", 6) == 0)
             p += 6;
 
-        int n = search_in_struct_unions(ctx, p, strlen(p));
+        int n = search_in_struct_unions(&builder->ctx, p, strlen(p));
         if (n < 0)
             Py_FatalError("lost a struct/union!");
 
-        const struct _cffi_struct_union_s *s = &ctx->struct_unions[n];
-        const struct _cffi_field_s *fld = &ctx->fields[s->first_field_index];
+        const struct _cffi_struct_union_s *s = &builder->ctx.struct_unions[n];
+        const struct _cffi_field_s *fld =
+            &builder->ctx.fields[s->first_field_index];
 
         /* XXX painfully build all the Python objects that are the args
            to b_complete_struct_or_union() */
@@ -300,7 +414,8 @@
             switch (_CFFI_GETOP(op)) {
 
             case _CFFI_OP_NOOP:
-                ctf = realize_c_type(ctx, ctx->types, _CFFI_GETARG(op));
+                ctf = realize_c_type(builder, builder->ctx.types,
+                                     _CFFI_GETARG(op));
                 break;
 
             default:
@@ -332,7 +447,7 @@
         ct->ct_flags &= ~CT_IS_OPAQUE;
         Py_DECREF(args);
         if (res == NULL) {
-            ct->ct_extra = (void *)ctx;
+            ct->ct_extra = builder;
             return -1;
         }
 
diff --git a/new/test_ffi_obj.py b/new/test_ffi_obj.py
--- a/new/test_ffi_obj.py
+++ b/new/test_ffi_obj.py
@@ -19,3 +19,19 @@
 
 def test_ffi_no_argument():
     py.test.raises(TypeError, _cffi1_backend.FFI, 42)
+
+def test_ffi_cache_type():
+    ffi = _cffi1_backend.FFI()
+    t1 = ffi.typeof("int **")
+    t2 = ffi.typeof("int *")
+    assert t2.item is t1.item.item
+    assert t2 is t1.item
+    assert ffi.typeof("int[][10]") is ffi.typeof("int[][10]")
+    assert ffi.typeof("int(*)()") is ffi.typeof("int(*)()")
+
+def test_ffi_cache_type_globally():
+    ffi1 = _cffi1_backend.FFI()
+    ffi2 = _cffi1_backend.FFI()
+    t1 = ffi1.typeof("int *")
+    t2 = ffi2.typeof("int *")
+    assert t1 is t2
diff --git a/new/test_recompiler.py b/new/test_recompiler.py
--- a/new/test_recompiler.py
+++ b/new/test_recompiler.py
@@ -173,14 +173,31 @@
     ffi = FFI()
     ffi.cdef("""struct foo_s { int b; short a; };
                 struct bar_s { struct foo_s *f; };""")
-    lib = verify(ffi, 'test_verify_struct',
+    ffi, lib = verify2(ffi, 'test_verify_struct',
                  """struct foo_s { short a; int b; };
                     struct bar_s { struct foo_s *f; };""")
+    ffi.typeof("struct bar_s *")
     p = ffi.new("struct foo_s *", {'a': -32768, 'b': -2147483648})
     assert p.a == -32768
     assert p.b == -2147483648
     py.test.raises(OverflowError, "p.a -= 1")
     py.test.raises(OverflowError, "p.b -= 1")
-    py.test.skip("XXX in-progress:")
     q = ffi.new("struct bar_s *", {'f': p})
     assert q.f == p
+
+def test_type_caching():
+    ffi1 = FFI(); ffi1.cdef("struct foo_s;")
+    ffi2 = FFI(); ffi2.cdef("struct foo_s;")    # different one!
+    ffi1, lib1 = verify2(ffi1, 'test_type_caching_1', 'struct foo_s;')
+    ffi2, lib2 = verify2(ffi2, 'test_type_caching_2', 'struct foo_s;')
+    # shared types
+    assert ffi1.typeof("long") is ffi2.typeof("long")
+    assert ffi1.typeof("long**") is ffi2.typeof("long * *")
+    assert ffi1.typeof("long(*)(int, ...)") is ffi2.typeof("long(*)(int, ...)")
+    # non-shared types
+    assert ffi1.typeof("struct foo_s") is not ffi2.typeof("struct foo_s")
+    assert ffi1.typeof("struct foo_s *") is not ffi2.typeof("struct foo_s *")
+    assert ffi1.typeof("struct foo_s*(*)()") is not (
+        ffi2.typeof("struct foo_s*(*)()"))
+    assert ffi1.typeof("void(*)(struct foo_s*)") is not (
+        ffi2.typeof("void(*)(struct foo_s*)"))

From noreply at buildbot.pypy.org  Fri Apr 17 16:48:27 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 17 Apr 2015 16:48:27 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: test and fix
Message-ID: <20150417144827.48BC01C051C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1740:0feb2cc57dcb
Date: 2015-04-17 16:49 +0200
http://bitbucket.org/cffi/cffi/changeset/0feb2cc57dcb/

Log:	test and fix

diff --git a/new/realize_c_type.c b/new/realize_c_type.c
--- a/new/realize_c_type.c
+++ b/new/realize_c_type.c
@@ -60,6 +60,8 @@
        decrefed, and 'y' never does.
     */
     CTypeDescrObject *ct = (CTypeDescrObject *)x;
+    if (ct == NULL)
+        return NULL;
 
     /* XXX maybe change the type of ct_name to be a real 'PyObject *'? */
     PyObject *name = PyString_FromString(ct->ct_name);
diff --git a/new/test_ffi_obj.py b/new/test_ffi_obj.py
--- a/new/test_ffi_obj.py
+++ b/new/test_ffi_obj.py
@@ -35,3 +35,8 @@
     t1 = ffi1.typeof("int *")
     t2 = ffi2.typeof("int *")
     assert t1 is t2
+
+def test_ffi_invalid():
+    ffi = _cffi1_backend.FFI()
+    # array of 10 times an "int[]" is invalid
+    py.test.raises(ValueError, ffi.typeof, "int[10][]")

From noreply at buildbot.pypy.org  Fri Apr 17 17:06:57 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 17 Apr 2015 17:06:57 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Copy docstrings from api.py.
Message-ID: <20150417150657.2AF4B1C1027@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1741:f976d53826d0
Date: 2015-04-17 17:07 +0200
http://bitbucket.org/cffi/cffi/changeset/f976d53826d0/

Log:	Copy docstrings from api.py.

diff --git a/new/ffi_obj.c b/new/ffi_obj.c
--- a/new/ffi_obj.c
+++ b/new/ffi_obj.c
@@ -168,6 +168,10 @@
     }
 }
 
+PyDoc_STRVAR(ffi_sizeof_doc,
+"Return the size in bytes of the argument.\n"
+"It can be a string naming a C type, or a 'cdata' instance.");
+
 static PyObject *ffi_sizeof(FFIObject *self, PyObject *arg)
 {
     CTypeDescrObject *ct = _ffi_type(self, arg, ACCEPT_ALL);
@@ -182,6 +186,11 @@
     return PyInt_FromSsize_t(ct->ct_size);
 }
 
+PyDoc_STRVAR(ffi_typeof_doc,
+"Parse the C type given as a string and return the\n"
+"corresponding  object.\n"
+"It can also be used on 'cdata' instance to get its C type.");
+
 static PyObject *ffi_typeof(FFIObject *self, PyObject *arg)
 {
     PyObject *x = (PyObject *)_ffi_type(self, arg, ACCEPT_STRING|ACCEPT_CDATA);
@@ -194,6 +203,27 @@
     return x;
 }
 
+PyDoc_STRVAR(ffi_new_doc,
+"Allocate an instance according to the specified C type and return a\n"
+"pointer to it.  The specified C type must be either a pointer or an\n"
+"array: ``new('X *')`` allocates an X and returns a pointer to it,\n"
+"whereas ``new('X[n]')`` allocates an array of n X'es and returns an\n"
+"array referencing it (which works mostly like a pointer, like in C).\n"
+"You can also use ``new('X[]', n)`` to allocate an array of a\n"
+"non-constant length n.\n"
+"\n"
+"The memory is initialized following the rules of declaring a global\n"
+"variable in C: by default it is zero-initialized, but an explicit\n"
+"initializer can be given which can be used to fill all or part of the\n"
+"memory.\n"
+"\n"
+"When the returned  object goes out of scope, the memory is\n"
+"freed.  In other words the returned  object has ownership of\n"
+"the value of type 'cdecl' that it points to.  This means that the raw\n"
+"data can be used as long as this object is kept alive, but must not be\n"
+"used for a longer time.  Be careful about that when copying the\n"
+"pointer to the memory somewhere else, e.g. into another structure.");
+
 static PyObject *ffi_new(FFIObject *self, PyObject *args)
 {
     CTypeDescrObject *ct;
@@ -550,15 +580,15 @@
     {"load_library",  (PyCFunction)ffi_load_library,METH_VARARGS|METH_KEYWORDS},
     {"offsetof",      (PyCFunction)ffi_offsetof,  METH_VARARGS},
 #endif
-    {"new",           (PyCFunction)ffi_new,       METH_VARARGS},
+    {"new",           (PyCFunction)ffi_new,       METH_VARARGS, ffi_new_doc},
 #if 0
     {"new_handle",    (PyCFunction)ffi_new_handle,METH_O},
 #endif
-    {"sizeof",        (PyCFunction)ffi_sizeof,    METH_O},
+    {"sizeof",        (PyCFunction)ffi_sizeof,    METH_O, ffi_sizeof_doc},
 #if 0
     {"string",        (PyCFunction)ffi_string,    METH_VARARGS},
 #endif
-    {"typeof",        (PyCFunction)ffi_typeof,    METH_O},
+    {"typeof",        (PyCFunction)ffi_typeof,    METH_O, ffi_typeof_doc},
     {NULL}
 };
 
diff --git a/new/test_ffi_obj.py b/new/test_ffi_obj.py
--- a/new/test_ffi_obj.py
+++ b/new/test_ffi_obj.py
@@ -40,3 +40,13 @@
     ffi = _cffi1_backend.FFI()
     # array of 10 times an "int[]" is invalid
     py.test.raises(ValueError, ffi.typeof, "int[10][]")
+
+def test_ffi_docstrings():
+    # check that all methods of the FFI class have a docstring.
+    check_type = type(_cffi1_backend.FFI.new)
+    for methname in dir(_cffi1_backend.FFI):
+        if not methname.startswith('_'):
+            method = getattr(_cffi1_backend.FFI, methname)
+            if isinstance(method, check_type):
+                assert method.__doc__, "method FFI.%s() has no docstring" % (
+                    methname,)

From noreply at buildbot.pypy.org  Fri Apr 17 17:09:20 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 17 Apr 2015 17:09:20 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof2: ups
Message-ID: <20150417150920.2D3DF1C1027@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof2
Changeset: r76820:49daf024add4
Date: 2015-04-17 17:09 +0200
http://bitbucket.org/pypy/pypy/changeset/49daf024add4/

Log:	ups

diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -200,9 +200,9 @@
         return fail_descr, target
 
     def enter_portal_frame(self, op):
-        self.codemap_builder.debug_merge_point(op.getarg(0).getint(),
-                                               op.getarg(1).getint(),
-                                               self.mc.get_relative_pos())
+        self.codemap_builder.enter_portal_frame(op.getarg(0).getint(),
+                                                op.getarg(1).getint(),
+                                                self.mc.get_relative_pos())
 
     def leave_portal_frame(self, op):
         self.codemap_builder.leave_portal_frame(op.getarg(0).getint(),

From noreply at buildbot.pypy.org  Fri Apr 17 17:13:54 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 17 Apr 2015 17:13:54 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Found out a "cleaner" way to hack for
	verify()
Message-ID: <20150417151354.896A61C1230@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1742:84244c3a7537
Date: 2015-04-17 17:14 +0200
http://bitbucket.org/cffi/cffi/changeset/84244c3a7537/

Log:	Found out a "cleaner" way to hack for verify()

diff --git a/new/ffi_obj.c b/new/ffi_obj.c
--- a/new/ffi_obj.c
+++ b/new/ffi_obj.c
@@ -536,40 +536,7 @@
 }
 #endif
 
-static PyObject *ffi__verified(FFIObject *self, PyObject *args)
-{
-    FFIObject *srcffi;
-
-    if (!PyArg_ParseTuple(args, "O!:_verified", &FFI_Type, &srcffi))
-        return NULL;
-
-    if (!srcffi->ctx_is_static)
-        goto invalid;
-
-    if (self->ctx_is_static)
-        goto invalid;
-
-    size_t i;
-    const char *p = (const char *)self->info.ctx;
-    for (i = 0; i < sizeof(struct _cffi_type_context_s); i++) {
-        if (*p++ != '\0')
-            goto invalid;
-    }
-
-    PyMem_Free((void *)self->info.ctx);
-    self->ctx_is_static = 1;
-    self->info.ctx = srcffi->info.ctx;
-
-    Py_INCREF(Py_None);
-    return Py_None;
-
- invalid:
-    PyErr_SetString(PyExc_ValueError, "XXX invalid source or destination");
-    return NULL;
-}
-
 static PyMethodDef ffi_methods[] = {
-    {"_verified",     (PyCFunction)ffi__verified, METH_VARARGS},
 #if 0
     {"addressof",     (PyCFunction)ffi_addressof, METH_VARARGS},
     {"cast",          (PyCFunction)ffi_cast,      METH_VARARGS},
diff --git a/new/recompiler.py b/new/recompiler.py
--- a/new/recompiler.py
+++ b/new/recompiler.py
@@ -558,15 +558,18 @@
     outputfilename = ffiplatform.compile(tmpdir, ext)
     return outputfilename
 
-def verify2(ffi, module_name, preamble, *args, **kwds):
+def verify(ffi, module_name, preamble, *args, **kwds):
     import imp
     assert module_name not in sys.modules, "module name conflict: %r" % (
         module_name,)
     outputfilename = recompile(ffi, module_name, preamble, *args, **kwds)
     module = imp.load_dynamic(module_name, outputfilename)
-    return module.ffi, module.lib
-
-def verify(ffi, module_name, preamble, *args, **kwds):
-    ffi2, lib = verify2(ffi, module_name, preamble, *args, **kwds)
-    ffi._verified(ffi2)
-    return lib
+    #
+    # hack hack hack: copy all *bound methods* from module.ffi back to the
+    # ffi instance.  Then calls like ffi.new() will invoke module.ffi.new().
+    for name in dir(module.ffi):
+        if not name.startswith('_'):
+            attr = getattr(module.ffi, name)
+            if attr is not getattr(ffi, name):
+                setattr(ffi, name, attr)
+    return module.lib
diff --git a/new/test_recompiler.py b/new/test_recompiler.py
--- a/new/test_recompiler.py
+++ b/new/test_recompiler.py
@@ -1,5 +1,5 @@
 import py
-from recompiler import Recompiler, verify, verify2
+from recompiler import Recompiler, verify
 from cffi1 import FFI
 
 
@@ -97,8 +97,8 @@
 def test_verify_typedef():
     ffi = FFI()
     ffi.cdef("typedef int **foo_t;")
-    ffi2, lib = verify2(ffi, 'test_verify_typedef', 'typedef int **foo_t;')
-    assert ffi2.sizeof("foo_t") == ffi.sizeof("void *")
+    lib = verify(ffi, 'test_verify_typedef', 'typedef int **foo_t;')
+    assert ffi.sizeof("foo_t") == ffi.sizeof("void *")
 
 def test_global_var_int():
     ffi = FFI()
@@ -160,20 +160,20 @@
 def test_verify_opaque_struct():
     ffi = FFI()
     ffi.cdef("struct foo_s;")
-    ffi, lib = verify2(ffi, 'test_verify_opaque_struct', "struct foo_s;")
+    lib = verify(ffi, 'test_verify_opaque_struct', "struct foo_s;")
     assert ffi.typeof("struct foo_s").cname == "struct foo_s"
 
 def test_verify_opaque_union():
     ffi = FFI()
     ffi.cdef("union foo_s;")
-    ffi, lib = verify2(ffi, 'test_verify_opaque_union', "union foo_s;")
+    lib = verify(ffi, 'test_verify_opaque_union', "union foo_s;")
     assert ffi.typeof("union foo_s").cname == "union foo_s"
 
 def test_verify_struct():
     ffi = FFI()
     ffi.cdef("""struct foo_s { int b; short a; };
                 struct bar_s { struct foo_s *f; };""")
-    ffi, lib = verify2(ffi, 'test_verify_struct',
+    lib = verify(ffi, 'test_verify_struct',
                  """struct foo_s { short a; int b; };
                     struct bar_s { struct foo_s *f; };""")
     ffi.typeof("struct bar_s *")
@@ -188,8 +188,8 @@
 def test_type_caching():
     ffi1 = FFI(); ffi1.cdef("struct foo_s;")
     ffi2 = FFI(); ffi2.cdef("struct foo_s;")    # different one!
-    ffi1, lib1 = verify2(ffi1, 'test_type_caching_1', 'struct foo_s;')
-    ffi2, lib2 = verify2(ffi2, 'test_type_caching_2', 'struct foo_s;')
+    lib1 = verify(ffi1, 'test_type_caching_1', 'struct foo_s;')
+    lib2 = verify(ffi2, 'test_type_caching_2', 'struct foo_s;')
     # shared types
     assert ffi1.typeof("long") is ffi2.typeof("long")
     assert ffi1.typeof("long**") is ffi2.typeof("long * *")

From noreply at buildbot.pypy.org  Fri Apr 17 17:39:40 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 17 Apr 2015 17:39:40 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: next test and fix
Message-ID: <20150417153940.A35771C0EDD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1743:1a738a27aa65
Date: 2015-04-17 17:40 +0200
http://bitbucket.org/cffi/cffi/changeset/1a738a27aa65/

Log:	next test and fix

diff --git a/new/_cffi_include.h b/new/_cffi_include.h
--- a/new/_cffi_include.h
+++ b/new/_cffi_include.h
@@ -155,6 +155,10 @@
 
 static void *_cffi_exports[_CFFI_NUM_EXPORTS];
 
+#define _cffi_type(index)   (                           \
+    assert((((uintptr_t)_cffi_types[index]) & 1) == 0), \
+    (CTypeDescrObject *)_cffi_types[index])
+
 
 static int _cffi_init(void)
 {
diff --git a/new/ffi_obj.c b/new/ffi_obj.c
--- a/new/ffi_obj.c
+++ b/new/ffi_obj.c
@@ -135,18 +135,22 @@
         }
         CTypeDescrObject *ct = realize_c_type(ffi->types_builder,
                                               ffi->info.output, index);
-        if (ct != NULL) {
-            /* Cache under the name given by 'arg', in addition to the
-               fact that the same ct is probably already cached under
-               its standardized name.  In a few cases, it is not, e.g.
-               if it is a primitive; for the purpose of this function,
-               the important point is the following line, which makes
-               sure that in any case the next _ffi_type() with the same
-               'arg' will succeed early, in PyDict_GetItem() above.
-            */
-            if (PyDict_SetItem(types_dict, arg, (PyObject *)ct) < 0)
-                return NULL;
-        }
+        if (ct == NULL)
+            return NULL;
+
+        /* Cache under the name given by 'arg', in addition to the
+           fact that the same ct is probably already cached under
+           its standardized name.  In a few cases, it is not, e.g.
+           if it is a primitive; for the purpose of this function,
+           the important point is the following line, which makes
+           sure that in any case the next _ffi_type() with the same
+           'arg' will succeed early, in PyDict_GetItem() above.
+        */
+        int err = PyDict_SetItem(types_dict, arg, (PyObject *)ct);
+        Py_DECREF(ct);   /* we know it was written in types_dict (unless we got
+                     out of memory), so there is at least this reference left */
+        if (err < 0)
+            return NULL;
         return ct;
     }
     else if ((accept & ACCEPT_CTYPE) && CTypeDescr_Check(arg)) {
diff --git a/new/lib_obj.c b/new/lib_obj.c
--- a/new/lib_obj.c
+++ b/new/lib_obj.c
@@ -43,6 +43,32 @@
                                         const struct _cffi_global_s *g,
                                         const char *s, int flags)
 {
+    /* First make sure the argument types and return type are really
+       built.  The C extension code can then assume that they are,
+       by calling _cffi_type().
+    */
+    CTypeDescrObject *ct;
+    int type_index = _CFFI_GETARG(g->type_op);
+    _cffi_opcode_t *opcodes = lib->l_types_builder->ctx.types;
+    assert(_CFFI_GETOP(opcodes[type_index]) == _CFFI_OP_FUNCTION);
+
+    /* return type: */
+    ct = realize_c_type(lib->l_types_builder, opcodes,
+                        _CFFI_GETARG(opcodes[type_index]));
+    if (ct == NULL)
+        return NULL;
+    Py_DECREF(ct);
+
+    /* argument types: */
+    int i = type_index + 1;
+    while (_CFFI_GETOP(opcodes[i]) != _CFFI_OP_FUNCTION_END) {
+        ct = realize_c_type(lib->l_types_builder, opcodes, i);
+        if (ct == NULL)
+            return NULL;
+        Py_DECREF(ct);
+        i++;
+    }
+
     /* xxx the few bytes of memory we allocate here leak, but it's a
        minor concern because it should only occur for CPYTHON_BLTN.
        There is one per real C function in a CFFI C extension module.
@@ -60,7 +86,7 @@
         goto no_memory;
 
     xfunc->types_builder = lib->l_types_builder;
-    xfunc->type_index = _CFFI_GETARG(g->type_op);
+    xfunc->type_index = type_index;
 
     return PyCFunction_NewEx(&xfunc->md, NULL, lib->l_libname);
 
diff --git a/new/recompiler.py b/new/recompiler.py
--- a/new/recompiler.py
+++ b/new/recompiler.py
@@ -215,7 +215,7 @@
     def _convert_funcarg_to_c_ptr_or_array(self, tp, fromvar, tovar, errcode):
         self._prnt('  datasize = _cffi_prepare_pointer_call_argument(')
         self._prnt('      _cffi_type(%d), %s, (char **)&%s);' % (
-            self._gettypenum(tp), fromvar, tovar))
+            self._typesdict[tp], fromvar, tovar))
         self._prnt('  if (datasize != 0) {')
         self._prnt('    if (datasize < 0)')
         self._prnt('      %s;' % errcode)
@@ -223,7 +223,7 @@
         self._prnt('    memset((void *)%s, 0, (size_t)datasize);' % (tovar,))
         self._prnt('    if (_cffi_convert_array_from_object('
                    '(char *)%s, _cffi_type(%d), %s) < 0)' % (
-            tovar, self._gettypenum(tp), fromvar))
+            tovar, self._typesdict[tp], fromvar))
         self._prnt('      %s;' % errcode)
         self._prnt('  }')
 
diff --git a/new/test_recompiler.py b/new/test_recompiler.py
--- a/new/test_recompiler.py
+++ b/new/test_recompiler.py
@@ -1,14 +1,19 @@
 import py
-from recompiler import Recompiler, verify
+import recompiler
 from cffi1 import FFI
 
 
 def check_type_table(input, expected_output):
     ffi = FFI()
     ffi.cdef(input)
-    recompiler = Recompiler(ffi, 'testmod')
-    recompiler.collect_type_table()
-    assert ''.join(map(str, recompiler.cffi_types)) == expected_output
+    recomp = recompiler.Recompiler(ffi, 'testmod')
+    recomp.collect_type_table()
+    assert ''.join(map(str, recomp.cffi_types)) == expected_output
+
+def verify(*args, **kwds):
+    kwds.setdefault('undef_macros', ['NDEBUG'])
+    return recompiler.verify(*args, **kwds)
+
 
 def test_type_table_func():
     check_type_table("double sin(double);",
@@ -86,6 +91,12 @@
     lib = verify(ffi, 'test_math_sin', '#include ')
     assert lib.cos(1.43) == math.cos(1.43)
 
+def test_funcarg_ptr():
+    ffi = FFI()
+    ffi.cdef("int foo(int *);")
+    lib = verify(ffi, 'test_funcarg_ptr', 'int foo(int *p) { return *p; }')
+    assert lib.foo([-12345]) == -12345
+
 def test_global_var_array():
     ffi = FFI()
     ffi.cdef("int a[100];")

From noreply at buildbot.pypy.org  Fri Apr 17 17:41:56 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 17 Apr 2015 17:41:56 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Test and fix
Message-ID: <20150417154156.EFEF51C0EDD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1744:15360a5f6128
Date: 2015-04-17 17:42 +0200
http://bitbucket.org/cffi/cffi/changeset/15360a5f6128/

Log:	Test and fix

diff --git a/new/recompiler.py b/new/recompiler.py
--- a/new/recompiler.py
+++ b/new/recompiler.py
@@ -97,6 +97,10 @@
     def _prnt(self, what=''):
         self._f.write(what + '\n')
 
+    def _gettypenum(self, type):
+        # a KeyError here is a bug.  please report it! :-)
+        return self._typesdict[type]
+
     def write_source_to_f(self, f, preamble):
         self._f = f
         prnt = self._prnt
@@ -215,7 +219,7 @@
     def _convert_funcarg_to_c_ptr_or_array(self, tp, fromvar, tovar, errcode):
         self._prnt('  datasize = _cffi_prepare_pointer_call_argument(')
         self._prnt('      _cffi_type(%d), %s, (char **)&%s);' % (
-            self._typesdict[tp], fromvar, tovar))
+            self._gettypenum(tp), fromvar, tovar))
         self._prnt('  if (datasize != 0) {')
         self._prnt('    if (datasize < 0)')
         self._prnt('      %s;' % errcode)
@@ -223,7 +227,7 @@
         self._prnt('    memset((void *)%s, 0, (size_t)datasize);' % (tovar,))
         self._prnt('    if (_cffi_convert_array_from_object('
                    '(char *)%s, _cffi_type(%d), %s) < 0)' % (
-            tovar, self._typesdict[tp], fromvar))
+            tovar, self._gettypenum(tp), fromvar))
         self._prnt('      %s;' % errcode)
         self._prnt('  }')
 
diff --git a/new/test_recompiler.py b/new/test_recompiler.py
--- a/new/test_recompiler.py
+++ b/new/test_recompiler.py
@@ -97,6 +97,13 @@
     lib = verify(ffi, 'test_funcarg_ptr', 'int foo(int *p) { return *p; }')
     assert lib.foo([-12345]) == -12345
 
+def test_funcres_ptr():
+    ffi = FFI()
+    ffi.cdef("int *foo(void);")
+    lib = verify(ffi, 'test_funcres_ptr',
+                 'int *foo(void) { static int x=-12345; return &x; }')
+    assert lib.foo()[0] == -12345
+
 def test_global_var_array():
     ffi = FFI()
     ffi.cdef("int a[100];")

From noreply at buildbot.pypy.org  Fri Apr 17 18:58:00 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 17 Apr 2015 18:58:00 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof2: fix that one
Message-ID: <20150417165800.7C4641C1589@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof2
Changeset: r76821:394f945d11d7
Date: 2015-04-17 18:57 +0200
http://bitbucket.org/pypy/pypy/changeset/394f945d11d7/

Log:	fix that one

diff --git a/rpython/jit/metainterp/test/support.py b/rpython/jit/metainterp/test/support.py
--- a/rpython/jit/metainterp/test/support.py
+++ b/rpython/jit/metainterp/test/support.py
@@ -36,6 +36,9 @@
             from rpython.rtyper.annlowlevel import llhelper
             return llhelper(FUNCPTR, func)
 
+        def get_unique_id(self, *args):
+            return 0
+
         def get_location_str(self, args):
             return 'location'
 

From noreply at buildbot.pypy.org  Fri Apr 17 19:01:20 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 17 Apr 2015 19:01:20 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof2: remove this assert,
	unique_id can be zero
Message-ID: <20150417170120.5CB631C1589@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof2
Changeset: r76822:f2a15c1a7c15
Date: 2015-04-17 19:01 +0200
http://bitbucket.org/pypy/pypy/changeset/f2a15c1a7c15/

Log:	remove this assert, unique_id can be zero

diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py
--- a/rpython/jit/backend/llsupport/codemap.py
+++ b/rpython/jit/backend/llsupport/codemap.py
@@ -176,10 +176,10 @@
             pos = self.patch_position.pop()
             self.l[pos] = size
             self.l[pos + 1] = len(self.l)
-        # at the end there should be no zeros
+        # at the end there should be no zeros, except unique_id which can
+        # be zero
         for i in range(len(self.l) / 4):
             item = self.l[i * 4] # unique_id
-            assert item > 0 # no zeros here
             item = self.l[i * 4 + 2] # end in asm
             assert item > 0
             item = self.l[i * 4 + 3] # end in l

From noreply at buildbot.pypy.org  Fri Apr 17 19:22:03 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 17 Apr 2015 19:22:03 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof2: make the test work,
	now it fails (but works)
Message-ID: <20150417172203.E78411C1027@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof2
Changeset: r76823:7333776ac58d
Date: 2015-04-17 19:22 +0200
http://bitbucket.org/pypy/pypy/changeset/7333776ac58d/

Log:	make the test work, now it fails (but works)

diff --git a/rpython/jit/backend/x86/test/test_recursive.py b/rpython/jit/backend/x86/test/test_recursive.py
--- a/rpython/jit/backend/x86/test/test_recursive.py
+++ b/rpython/jit/backend/x86/test/test_recursive.py
@@ -1,4 +1,5 @@
 
+import cffi
 from rpython.jit.metainterp.test.test_recursive import RecursiveTests
 from rpython.jit.backend.x86.test.test_basic import Jit386Mixin
 from rpython.jit.backend.llsupport import codemap
@@ -8,16 +9,14 @@
 class TestRecursive(Jit386Mixin, RecursiveTests):
     # for the individual tests see
     # ====> ../../../metainterp/test/test_recursive.py
-    def check_get_unique_id(self):
+    def check_get_unique_id(self, codemaps):
         if WORD == 4:
             return # this is 64 bit only check
-        jit_codemap = codemap._codemap.jit_codemap
-        codemaps = [(jit_codemap[i].addr, jit_codemap[i].machine_code_size)
-                    for i in range(3)]
-        assert codemap._codemap.jit_codemap_used == 3
+
+        assert len(codemaps) == 3
         codemaps.sort(lambda a, b: cmp(a[1], b[1]))
         # biggest is the big loop, smallest is the bridge
-        assert unpack_traceback(codemaps[1][0]) == []
+        assert unpack_traceback(codemaps[1][0]) == [2]
         # XXX very specific ASM addresses, very fragile test, but what we can
         #     do, really? 64bit only so far
         assert unpack_traceback(codemaps[0][0]) == [2]
diff --git a/rpython/jit/metainterp/test/test_recursive.py b/rpython/jit/metainterp/test/test_recursive.py
--- a/rpython/jit/metainterp/test/test_recursive.py
+++ b/rpython/jit/metainterp/test/test_recursive.py
@@ -7,6 +7,7 @@
 from rpython.jit.codewriter.policy import StopAtXPolicy
 from rpython.rtyper.annlowlevel import hlstr
 from rpython.jit.metainterp.warmspot import get_stats
+from rpython.jit.backend.llsupport import codemap
 
 class RecursiveTests:
 
@@ -1297,27 +1298,38 @@
         self.check_trace_count_at_most(2)   # and not, e.g., 24
 
     def test_get_unique_id(self):
-        def get_unique_id(pc, code):
-            return (code + 1) * 2
+        lst = []
         
-        driver = JitDriver(greens=["pc", "code"], reds='auto',
-                           get_unique_id=get_unique_id)
+        def reg_codemap(self, (start, size, l)):
+            lst.append((start, size))
+            old_reg_codemap(self, (start, size, l))
+        
+        old_reg_codemap = codemap.CodemapStorage.register_codemap
+        try:
+            codemap.CodemapStorage.register_codemap = reg_codemap
+            def get_unique_id(pc, code):
+                return (code + 1) * 2
 
-        def f(pc, code):
-            i = 0
-            while i < 10:
-                driver.jit_merge_point(pc=pc, code=code)
-                pc += 1
-                if pc == 3:
-                    if code == 1:
-                        f(0, 0)
-                    pc = 0
-                i += 1
+            driver = JitDriver(greens=["pc", "code"], reds='auto',
+                               get_unique_id=get_unique_id)
 
-        self.meta_interp(f, [0, 1], inline=True)
-        self.check_get_unique_id() # overloaded on assembler backends
+            def f(pc, code):
+                i = 0
+                while i < 10:
+                    driver.jit_merge_point(pc=pc, code=code)
+                    pc += 1
+                    if pc == 3:
+                        if code == 1:
+                            f(0, 0)
+                        pc = 0
+                    i += 1
 
-    def check_get_unique_id(self):
+            self.meta_interp(f, [0, 1], inline=True)
+            self.check_get_unique_id(lst) # overloaded on assembler backends
+        finally:
+            codemap.CodemapStorage.register_codemap = old_reg_codemap
+
+    def check_get_unique_id(self, lst):
         pass
 
 class TestLLtype(RecursiveTests, LLJitMixin):

From noreply at buildbot.pypy.org  Fri Apr 17 19:26:57 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 17 Apr 2015 19:26:57 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof2: kill unusued imports
Message-ID: <20150417172657.028E61C1174@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof2
Changeset: r76824:fa908efd2d01
Date: 2015-04-17 19:26 +0200
http://bitbucket.org/pypy/pypy/changeset/fa908efd2d01/

Log:	kill unusued imports

diff --git a/rpython/jit/backend/x86/test/test_recursive.py b/rpython/jit/backend/x86/test/test_recursive.py
--- a/rpython/jit/backend/x86/test/test_recursive.py
+++ b/rpython/jit/backend/x86/test/test_recursive.py
@@ -1,8 +1,6 @@
 
-import cffi
 from rpython.jit.metainterp.test.test_recursive import RecursiveTests
 from rpython.jit.backend.x86.test.test_basic import Jit386Mixin
-from rpython.jit.backend.llsupport import codemap
 from rpython.jit.backend.llsupport.codemap import unpack_traceback
 from rpython.jit.backend.x86.arch import WORD
 

From noreply at buildbot.pypy.org  Sat Apr 18 11:37:48 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 18 Apr 2015 11:37:48 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Deprecate the logic to build types
 directly from the cffi/ pure python package
Message-ID: <20150418093748.BB41E1C0755@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1745:2b08ac02f836
Date: 2015-04-18 11:30 +0200
http://bitbucket.org/cffi/cffi/changeset/2b08ac02f836/

Log:	Deprecate the logic to build types directly from the cffi/ pure
	python package

diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -2,6 +2,9 @@
 from .lock import allocate_lock
 import _cffi1_backend
 
+class DeprecatedError(Exception):
+    pass
+
 try:
     callable
 except NameError:
@@ -70,9 +73,9 @@
             if name.startswith('RTLD_'):
                 setattr(self, name, getattr(backend, name))
         #
-        with self._lock:
-            self.BVoidP = self._get_cached_btype(model.voidp_type)
-            self.BCharA = self._get_cached_btype(model.char_array_type)
+        #with self._lock:
+        #    self.BVoidP = self._get_cached_btype(model.voidp_type)
+        #    self.BCharA = self._get_cached_btype(model.char_array_type)
 
     def cdef(self, csource, override=False, packed=False):
         """Parse the given C source.  This registers all declared functions,
@@ -314,16 +317,7 @@
             return gc_weakrefs.build(cdata, destructor)
 
     def _get_cached_btype(self, type):
-        assert self._lock.acquire(False) is False
-        # call me with the lock!
-        try:
-            BType = self._cached_btypes[type]
-        except KeyError:
-            finishlist = []
-            BType = type.get_cached_btype(self, finishlist)
-            for type in finishlist:
-                type.finish_backend_type(self, finishlist)
-        return BType
+        raise DeprecatedError
 
     def verify(self, source='', tmpdir=None, **kwargs):
         """Verify that the current ffi signatures compile on this
diff --git a/new/test_dlopen.py b/new/test_dlopen.py
--- a/new/test_dlopen.py
+++ b/new/test_dlopen.py
@@ -1,8 +1,10 @@
+import py
 from cffi1 import FFI
 import math
 
 
 def test_math_sin():
+    py.test.skip("XXX redo!")
     ffi = FFI()
     ffi.cdef("double sin(double);")
     m = ffi.dlopen('m')

From noreply at buildbot.pypy.org  Sat Apr 18 11:37:49 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 18 Apr 2015 11:37:49 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Prebuilt FFI.NULL
Message-ID: <20150418093749.AB8CE1C0755@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1746:182cb9f15913
Date: 2015-04-18 11:38 +0200
http://bitbucket.org/cffi/cffi/changeset/182cb9f15913/

Log:	Prebuilt FFI.NULL

diff --git a/new/cffi1_module.c b/new/cffi1_module.c
--- a/new/cffi1_module.c
+++ b/new/cffi1_module.c
@@ -20,6 +20,8 @@
         return -1;
     if (!PyType_Ready(&Lib_Type) < 0)
         return -1;
+    if (init_global_types_dict(FFI_Type.tp_dict) < 0)
+        return -1;
 
     FFIError = PyErr_NewException("ffi.error", NULL, NULL);
     if (FFIError == NULL)
diff --git a/new/realize_c_type.c b/new/realize_c_type.c
--- a/new/realize_c_type.c
+++ b/new/realize_c_type.c
@@ -6,13 +6,42 @@
 
 
 static PyObject *all_primitives[_CFFI__NUM_PRIM];
+static PyObject *global_types_dict;
 
-static PyObject *fetch_global_types_dict(void)
+static PyObject *build_primitive_type(int num);   /* forward */
+
+static int init_global_types_dict(PyObject *ffi_type_dict)
 {
-    static PyObject *result = NULL;
-    if (!result)
-        result = PyDict_New();
-    return result;
+    int err;
+    PyObject *ct, *ct2, *pnull;
+
+    global_types_dict = PyDict_New();
+    if (global_types_dict == NULL)
+        return -1;
+
+    ct = build_primitive_type(_CFFI_PRIM_VOID);         // 'void'
+    if (ct == NULL)
+        return -1;
+    if (PyDict_SetItemString(global_types_dict,
+                             ((CTypeDescrObject *)ct)->ct_name, ct) < 0) {
+        return -1;
+    }
+    ct2 = new_pointer_type((CTypeDescrObject *)ct);     // 'void *'
+    if (ct2 == NULL)
+        return -1;
+    if (PyDict_SetItemString(global_types_dict,
+                             ((CTypeDescrObject *)ct2)->ct_name, ct2) < 0) {
+        Py_DECREF(ct2);
+        return -1;
+    }
+
+    pnull = new_simple_cdata(NULL, (CTypeDescrObject *)ct2);
+    Py_DECREF(ct2);
+    if (pnull == NULL)
+        return -1;
+    err = PyDict_SetItemString(ffi_type_dict, "NULL", pnull);
+    Py_DECREF(pnull);
+    return err;
 }
 
 static void free_builder_c(builder_c_t *builder)
@@ -83,11 +112,7 @@
            ffi instances.  Look it up and possibly add it to the global
            types dict.
         */
-        PyObject *gdict = fetch_global_types_dict();
-        if (gdict == NULL)
-            goto no_memory;
-
-        y = PyDict_GetItem(gdict, name);
+        y = PyDict_GetItem(global_types_dict, name);
         if (y != NULL) {
             Py_INCREF(y);
             Py_DECREF(x);
@@ -95,7 +120,7 @@
         }
         else {
             /* Not found in the global dictionary.  Put it there. */
-            if (PyDict_SetItem(gdict, name, x) < 0)
+            if (PyDict_SetItem(global_types_dict, name, x) < 0)
                 goto no_memory;
         }
     }
diff --git a/new/test_ffi_obj.py b/new/test_ffi_obj.py
--- a/new/test_ffi_obj.py
+++ b/new/test_ffi_obj.py
@@ -50,3 +50,7 @@
             if isinstance(method, check_type):
                 assert method.__doc__, "method FFI.%s() has no docstring" % (
                     methname,)
+
+def test_ffi_NULL():
+    NULL = _cffi1_backend.FFI.NULL
+    assert _cffi1_backend.FFI().typeof(NULL).cname == "void *"

From noreply at buildbot.pypy.org  Sat Apr 18 11:44:12 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 18 Apr 2015 11:44:12 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: ffi.string()
Message-ID: <20150418094412.A55921C0484@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1747:f12d17ae448d
Date: 2015-04-18 11:44 +0200
http://bitbucket.org/cffi/cffi/changeset/f12d17ae448d/

Log:	ffi.string()

diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -226,7 +226,7 @@
             cdecl = self._typeof(cdecl)
         return self._backend.cast(cdecl, source)
 
-    def string(self, cdata, maxlen=-1):
+    def XXXstring(self, cdata, maxlen=-1):
         """Return a Python string (or unicode string) from the 'cdata'.
         If 'cdata' is a pointer or array of characters or bytes, returns
         the null-terminated string.  The returned string extends until
diff --git a/new/ffi_obj.c b/new/ffi_obj.c
--- a/new/ffi_obj.c
+++ b/new/ffi_obj.c
@@ -256,69 +256,28 @@
 
     return do_cast(ct, ob);
 }
+#endif
 
-static PyObject *ffi_string(ZefFFIObject *self, PyObject *args)
-{
-    CDataObject *cd;
-    Py_ssize_t maxlen = -1;
-    if (!PyArg_ParseTuple(args, "O!|n:string",
-                          &CData_Type, &cd, &maxlen))
-        return NULL;
+PyDoc_STRVAR(ffi_string_doc,
+"Return a Python string (or unicode string) from the 'cdata'.  If\n"
+"'cdata' is a pointer or array of characters or bytes, returns the\n"
+"null-terminated string.  The returned string extends until the first\n"
+"null character, or at most 'maxlen' characters.  If 'cdata' is an\n"
+"array then 'maxlen' defaults to its length.\n"
+"\n"
+"If 'cdata' is a pointer or array of wchar_t, returns a unicode string\n"
+"following the same rules.\n"
+"\n"
+"If 'cdata' is a single character or byte or a wchar_t, returns it as a\n"
+"string or unicode string.\n"
+"\n"
+"If 'cdata' is an enum, returns the value of the enumerator as a\n"
+"string, or 'NUMBER' if the value is out of range.\n");
 
-    if (cd->c_type->ct_itemdescr != NULL &&
-        cd->c_type->ct_itemdescr->ct_flags & (CT_PRIMITIVE_CHAR |
-                                              CT_PRIMITIVE_SIGNED |
-                                              CT_PRIMITIVE_UNSIGNED)) {
-        Py_ssize_t length = maxlen;
-        if (cd->c_data == NULL) {
-            PyObject *s = cdata_repr(cd);
-            if (s != NULL) {
-                PyErr_Format(PyExc_RuntimeError,
-                             "cannot use string() on %s",
-                             PyText_AS_UTF8(s));
-                Py_DECREF(s);
-            }
-            return NULL;
-        }
-        if (length < 0 && cd->c_type->ct_flags & CT_ARRAY) {
-            length = get_array_length(cd);
-        }
-        if (cd->c_type->ct_itemdescr->ct_size == sizeof(char)) {
-            const char *start = cd->c_data;
-            if (length < 0) {
-                /*READ(start, 1)*/
-                length = strlen(start);
-                /*READ(start, length)*/
-            }
-            else {
-                const char *end;
-                /*READ(start, length)*/
-                end = (const char *)memchr(start, 0, length);
-                if (end != NULL)
-                    length = end - start;
-            }
-            return PyBytes_FromStringAndSize(start, length);
-        }
-    }
-    else if (cd->c_type->ct_flags & CT_IS_ENUM) {
-        abort();
-        //return convert_cdata_to_enum_string(cd, 0);
-    }
-    else if (cd->c_type->ct_flags & CT_IS_BOOL) {
-        /* fall through to TypeError */
-    }
-    else if (cd->c_type->ct_flags & (CT_PRIMITIVE_CHAR |
-                                     CT_PRIMITIVE_SIGNED |
-                                     CT_PRIMITIVE_UNSIGNED)) {
-        /*READ(cd->c_data, cd->c_type->ct_size)*/
-        if (cd->c_type->ct_size == sizeof(char))
-            return PyBytes_FromStringAndSize(cd->c_data, 1);
-    }
-    PyErr_Format(PyExc_TypeError, "string(): unexpected cdata '%s' argument",
-                 cd->c_type->ct_name);
-    return NULL;
-}
+#define ffi_string  b_string     /* ffi_string() => b_string()
+                                    from _cffi_backend.c */
 
+#if 0
 static CFieldObject *_ffi_field(CTypeDescrObject *ct, const char *fieldname)
 {
     CFieldObject *cf;
@@ -555,11 +514,9 @@
 #if 0
     {"new_handle",    (PyCFunction)ffi_new_handle,METH_O},
 #endif
-    {"sizeof",        (PyCFunction)ffi_sizeof,    METH_O, ffi_sizeof_doc},
-#if 0
-    {"string",        (PyCFunction)ffi_string,    METH_VARARGS},
-#endif
-    {"typeof",        (PyCFunction)ffi_typeof,    METH_O, ffi_typeof_doc},
+    {"sizeof",        (PyCFunction)ffi_sizeof,    METH_O,       ffi_sizeof_doc},
+    {"string",        (PyCFunction)ffi_string,    METH_VARARGS, ffi_string_doc},
+    {"typeof",        (PyCFunction)ffi_typeof,    METH_O,       ffi_typeof_doc},
     {NULL}
 };
 
diff --git a/new/test_ffi_obj.py b/new/test_ffi_obj.py
--- a/new/test_ffi_obj.py
+++ b/new/test_ffi_obj.py
@@ -54,3 +54,8 @@
 def test_ffi_NULL():
     NULL = _cffi1_backend.FFI.NULL
     assert _cffi1_backend.FFI().typeof(NULL).cname == "void *"
+
+def test_ffi_string():
+    ffi = _cffi1_backend.FFI()
+    p = ffi.new("char[]", "foobar\x00baz")
+    assert ffi.string(p) == "foobar"

From noreply at buildbot.pypy.org  Sat Apr 18 12:10:26 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 18 Apr 2015 12:10:26 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: the '[...]' syntax
Message-ID: <20150418101026.E228B1C0484@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1748:28722032dd52
Date: 2015-04-18 12:11 +0200
http://bitbucket.org/cffi/cffi/changeset/28722032dd52/

Log:	the '[...]' syntax

diff --git a/cffi/model.py b/cffi/model.py
--- a/cffi/model.py
+++ b/cffi/model.py
@@ -222,7 +222,7 @@
         elif length == '...':
             brackets = '&[/*...*/]'
         else:
-            brackets = '&[%d]' % length
+            brackets = '&[%s]' % length
         self.c_name_with_marker = (
             self.item.c_name_with_marker.replace('&', brackets))
 
diff --git a/new/_cffi_include.h b/new/_cffi_include.h
--- a/new/_cffi_include.h
+++ b/new/_cffi_include.h
@@ -159,6 +159,8 @@
     assert((((uintptr_t)_cffi_types[index]) & 1) == 0), \
     (CTypeDescrObject *)_cffi_types[index])
 
+#define _cffi_array_len(array)   (sizeof(array) / sizeof((array)[0]))
+
 
 static int _cffi_init(void)
 {
diff --git a/new/recompiler.py b/new/recompiler.py
--- a/new/recompiler.py
+++ b/new/recompiler.py
@@ -369,11 +369,19 @@
     # ----------
     # named structs or unions
 
+    def _field_type(self, tp_struct, field_name, tp_field):
+        if isinstance(tp_field, model.ArrayType) and tp_field.length == '...':
+            ptr_struct_name = tp_struct.get_c_name('*')
+            actual_length = '_cffi_array_len(((%s)0)->%s)' % (
+                ptr_struct_name, field_name)
+            tp_field = model.ArrayType(tp_field.item, actual_length)
+        return tp_field
+
     def _generate_cpy_struct_collecttype(self, tp, name):
         self._do_collect_type(tp)
         if tp.fldtypes is not None:
-            for tp1 in tp.fldtypes:
-                self._do_collect_type(tp1)
+            for name1, tp1 in zip(tp.fldnames, tp.fldtypes):
+                self._do_collect_type(self._field_type(tp, name1, tp1))
 
     def _generate_cpy_struct_decl(self, tp, name):
         if tp.fldtypes is not None:
@@ -390,6 +398,7 @@
         if tp.fldtypes is not None:
             c_field = [name]
             for fldname, fldtype in zip(tp.fldnames, tp.fldtypes):
+                fldtype = self._field_type(tp, fldname, fldtype)
                 spaces = " " * len(fldname)
                 c_field.append(
                     '  { "%s", offsetof(%s, %s),\n' % (
@@ -483,13 +492,20 @@
     # ----------
     # global variables
 
+    def _global_type(self, tp, global_name):
+        if isinstance(tp, model.ArrayType) and tp.length == '...':
+            actual_length = '_cffi_array_len(%s)' % (global_name,)
+            tp = model.ArrayType(tp.item, actual_length)
+        return tp
+
     def _generate_cpy_variable_collecttype(self, tp, name):
-        self._do_collect_type(tp)
+        self._do_collect_type(self._global_type(tp, name))
 
     def _generate_cpy_variable_decl(self, tp, name):
         pass
 
     def _generate_cpy_variable_ctx(self, tp, name):
+        tp = self._global_type(tp, name)
         type_index = self._typesdict[tp]
         self._lsts["global"].append(
             '  { "%s", &%s, _CFFI_OP(_CFFI_OP_GLOBAL_VAR, %d)},'
@@ -528,10 +544,15 @@
         item_index = self._typesdict[tp.item]
         if tp.length is None:
             self.cffi_types[index] = CffiOp(OP_OPEN_ARRAY, item_index)
+        elif tp.length == '...':
+            raise ffiplatform.VerificationError(
+                "type %s badly placed: the '...' array length can only be "
+                "used on global arrays or on fields of structures" % (
+                    str(tp).replace('/*...*/', '...'),))
         else:
             assert self.cffi_types[index + 1] == 'LEN'
             self.cffi_types[index] = CffiOp(OP_ARRAY, item_index)
-            self.cffi_types[index + 1] = CffiOp(None, '%d' % (tp.length,))
+            self.cffi_types[index + 1] = CffiOp(None, str(tp.length))
 
     def _emit_bytecode_StructType(self, tp, index):
         struct_index = self._struct_unions[tp]
diff --git a/new/test_recompiler.py b/new/test_recompiler.py
--- a/new/test_recompiler.py
+++ b/new/test_recompiler.py
@@ -219,3 +219,26 @@
         ffi2.typeof("struct foo_s*(*)()"))
     assert ffi1.typeof("void(*)(struct foo_s*)") is not (
         ffi2.typeof("void(*)(struct foo_s*)"))
+
+def test_dotdotdot_length_of_array_field():
+    ffi = FFI()
+    ffi.cdef("struct foo_s { int a[...]; int b[...]; };")
+    verify(ffi, 'test_dotdotdot_length_of_array_field',
+           "struct foo_s { int a[42]; int b[11]; };")
+    assert ffi.sizeof("struct foo_s") == (42 + 11) * 4
+    p = ffi.new("struct foo_s *")
+    assert p.a[41] == p.b[10] == 0
+    py.test.raises(IndexError, "p.a[42]")
+    py.test.raises(IndexError, "p.b[11]")
+
+def test_dotdotdot_global_array():
+    ffi = FFI()
+    ffi.cdef("int aa[...]; int bb[...];")
+    lib = verify(ffi, 'test_dotdotdot_global_array',
+                 "int aa[41]; int bb[12];")
+    assert ffi.sizeof(lib.aa) == 41 * 4
+    assert ffi.sizeof(lib.bb) == 12 * 4
+    assert lib.aa[40] == lib.bb[11] == 0
+    py.test.raises(IndexError, "lib.aa[41]")
+    py.test.raises(IndexError, "lib.bb[12]")
+

From noreply at buildbot.pypy.org  Sat Apr 18 12:11:54 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 18 Apr 2015 12:11:54 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Add the readdir2 demo
Message-ID: <20150418101154.6CCC31C0484@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1749:d72e8ca1cb4e
Date: 2015-04-18 12:12 +0200
http://bitbucket.org/cffi/cffi/changeset/d72e8ca1cb4e/

Log:	Add the readdir2 demo

diff --git a/new/readdir2.py b/new/readdir2.py
new file mode 100644
--- /dev/null
+++ b/new/readdir2.py
@@ -0,0 +1,33 @@
+# A Linux-only demo, using verify() instead of hard-coding the exact layouts
+#
+import sys
+from _readdir2 import ffi, lib
+
+if not sys.platform.startswith('linux'):
+    raise Exception("Linux-only demo")
+
+
+def walk(basefd, path):
+    print '{', path
+    dirfd = lib.openat(basefd, path, 0)
+    if dirfd < 0:
+        # error in openat()
+        return
+    dir = lib.fdopendir(dirfd)
+    dirent = ffi.new("struct dirent *")
+    result = ffi.new("struct dirent **")
+    while True:
+        if lib.readdir_r(dir, dirent, result):
+            # error in readdir_r()
+            break
+        if result[0] == ffi.NULL:
+            break
+        name = ffi.string(dirent.d_name)
+        print '%3d %s' % (dirent.d_type, name)
+        if dirent.d_type == lib.DT_DIR and name != '.' and name != '..':
+            walk(dirfd, name)
+    lib.closedir(dir)
+    print '}'
+
+
+walk(-1, "/tmp")
diff --git a/new/readdir2_build.py b/new/readdir2_build.py
new file mode 100644
--- /dev/null
+++ b/new/readdir2_build.py
@@ -0,0 +1,34 @@
+from cffi1 import FFI
+from recompiler import recompile
+
+ffi = FFI()
+ffi.cdef("""
+
+    typedef ... DIR;
+
+    struct dirent {
+        unsigned char  d_type;      /* type of file; not supported
+                                       by all file system types */
+        char           d_name[...]; /* filename */
+        ...;
+    };
+
+    int readdir_r(DIR *dirp, struct dirent *entry, struct dirent **result);
+    int openat(int dirfd, const char *pathname, int flags);
+    DIR *fdopendir(int fd);
+    int closedir(DIR *dirp);
+
+    static const int DT_DIR;
+
+""")
+recompile(ffi, "_readdir2", """
+#ifndef _ATFILE_SOURCE
+#  define _ATFILE_SOURCE
+#endif
+#ifndef _BSD_SOURCE
+#  define _BSD_SOURCE
+#endif
+#include 
+#include 
+#include 
+""")

From noreply at buildbot.pypy.org  Sat Apr 18 12:27:39 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 18 Apr 2015 12:27:39 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Check the 'field_size' of _cffi_field_s
Message-ID: <20150418102739.595D91C0755@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1750:2a81b5782307
Date: 2015-04-18 12:28 +0200
http://bitbucket.org/cffi/cffi/changeset/2a81b5782307/

Log:	Check the 'field_size' of _cffi_field_s

diff --git a/new/realize_c_type.c b/new/realize_c_type.c
--- a/new/realize_c_type.c
+++ b/new/realize_c_type.c
@@ -451,6 +451,15 @@
                 return -1;
             }
 
+            if (ctf->ct_size != fld->field_size) {
+                PyErr_Format(FFIError,
+                             "%s field '%s' was declared in the cdef to be"
+                             " %zd bytes, but is actually %zd bytes",
+                             ct->ct_name, fld->name,
+                             ctf->ct_size, fld->field_size);
+                return -1;
+            }
+
             f = Py_BuildValue("(sOin)", fld->name, ctf,
                               (int)-1, (Py_ssize_t)fld->field_offset);
             if (f == NULL) {
diff --git a/new/test_recompiler.py b/new/test_recompiler.py
--- a/new/test_recompiler.py
+++ b/new/test_recompiler.py
@@ -242,3 +242,13 @@
     py.test.raises(IndexError, "lib.aa[41]")
     py.test.raises(IndexError, "lib.bb[12]")
 
+def test_misdeclared_field_1():
+    ffi = FFI()
+    ffi.cdef("struct foo_s { int a[5]; };")
+    verify(ffi, 'test_misdeclared_field_1',
+           "struct foo_s { int a[6]; };")
+    assert ffi.sizeof("struct foo_s") == 24  # found by the actual C code
+    p = ffi.new("struct foo_s *")
+    e = py.test.raises(ffi.error, "p.a")     # lazily build the fields and boom
+    assert str(e.value) == ("struct foo_s field 'a' was declared in the "
+                            "cdef to be 20 bytes, but is actually 24 bytes")

From noreply at buildbot.pypy.org  Sat Apr 18 12:47:47 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 18 Apr 2015 12:47:47 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: ffi.errno
Message-ID: <20150418104747.592011C03F7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1751:a6bf54f3fe6e
Date: 2015-04-18 12:48 +0200
http://bitbucket.org/cffi/cffi/changeset/a6bf54f3fe6e/

Log:	ffi.errno

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -5139,12 +5139,16 @@
     return PyInt_FromLong(err);
 }
 
-static PyObject *b_set_errno(PyObject *self, PyObject *args)
-{
-    int i;
-    if (!PyArg_ParseTuple(args, "i:set_errno", &i))
+static PyObject *b_set_errno(PyObject *self, PyObject *arg)
+{
+    long ival = PyInt_AsLong(arg);
+    if (ival == -1 && PyErr_Occurred())
         return NULL;
-    errno = i;
+    else if (ival < INT_MIN || ival > INT_MAX) {
+        PyErr_SetString(PyExc_OverflowError, "errno value too large");
+        return NULL;
+    }
+    errno = (int)ival;
     save_errno_only();
     errno = 0;
     Py_INCREF(Py_None);
@@ -5626,7 +5630,7 @@
     {"string", b_string, METH_VARARGS},
     {"buffer", b_buffer, METH_VARARGS},
     {"get_errno", b_get_errno, METH_NOARGS},
-    {"set_errno", b_set_errno, METH_VARARGS},
+    {"set_errno", b_set_errno, METH_O},
     {"newp_handle", b_newp_handle, METH_VARARGS},
     {"from_handle", b_from_handle, METH_O},
     {"from_buffer", b_from_buffer, METH_VARARGS},
diff --git a/new/ffi_obj.c b/new/ffi_obj.c
--- a/new/ffi_obj.c
+++ b/new/ffi_obj.c
@@ -499,6 +499,24 @@
 }
 #endif
 
+PyDoc_STRVAR(ffi_errno_doc, "the value of 'errno' from/to the C calls");
+
+static PyObject *ffi_get_errno(PyObject *self, void *closure)
+{
+    /* xxx maybe think about how to make the saved errno local
+       to an ffi instance */
+    return b_get_errno(NULL, NULL);
+}
+
+static int ffi_set_errno(PyObject *self, PyObject *newval, void *closure)
+{
+    PyObject *x = b_set_errno(NULL, newval);
+    if (x == NULL)
+        return -1;
+    Py_DECREF(x);
+    return 0;
+}
+
 static PyMethodDef ffi_methods[] = {
 #if 0
     {"addressof",     (PyCFunction)ffi_addressof, METH_VARARGS},
@@ -520,6 +538,11 @@
     {NULL}
 };
 
+static PyGetSetDef ffi_getsets[] = {
+    {"errno",  ffi_get_errno,  ffi_set_errno,  ffi_errno_doc},
+    {NULL}
+};
+
 static PyTypeObject FFI_Type = {
     PyVarObject_HEAD_INIT(NULL, 0)
     "cffi.FFI",
@@ -551,7 +574,7 @@
     0,                                          /* tp_iternext */
     ffi_methods,                                /* tp_methods */
     0,                                          /* tp_members */
-    0,                                          /* tp_getset */
+    ffi_getsets,                                /* tp_getset */
     0,                                          /* tp_base */
     0,                                          /* tp_dict */
     0,                                          /* tp_descr_get */
diff --git a/new/test_ffi_obj.py b/new/test_ffi_obj.py
--- a/new/test_ffi_obj.py
+++ b/new/test_ffi_obj.py
@@ -59,3 +59,9 @@
     ffi = _cffi1_backend.FFI()
     p = ffi.new("char[]", "foobar\x00baz")
     assert ffi.string(p) == "foobar"
+
+def test_ffi_errno():
+    # xxx not really checking errno, just checking that we can read/write it
+    ffi = _cffi1_backend.FFI()
+    ffi.errno = 42
+    assert ffi.errno == 42

From noreply at buildbot.pypy.org  Sat Apr 18 12:53:26 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 18 Apr 2015 12:53:26 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: test and fix
Message-ID: <20150418105326.3B7D01C03F7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1752:ec0e49dc60d1
Date: 2015-04-18 12:54 +0200
http://bitbucket.org/cffi/cffi/changeset/ec0e49dc60d1/

Log:	test and fix

diff --git a/new/recompiler.py b/new/recompiler.py
--- a/new/recompiler.py
+++ b/new/recompiler.py
@@ -374,7 +374,7 @@
             ptr_struct_name = tp_struct.get_c_name('*')
             actual_length = '_cffi_array_len(((%s)0)->%s)' % (
                 ptr_struct_name, field_name)
-            tp_field = model.ArrayType(tp_field.item, actual_length)
+            tp_field = tp_field.resolve_length(actual_length)
         return tp_field
 
     def _generate_cpy_struct_collecttype(self, tp, name):
@@ -400,11 +400,15 @@
             for fldname, fldtype in zip(tp.fldnames, tp.fldtypes):
                 fldtype = self._field_type(tp, fldname, fldtype)
                 spaces = " " * len(fldname)
+                if (isinstance(fldtype, model.ArrayType) and
+                       fldtype.length is None):
+                    size = '-1'
+                else:
+                    size = 'sizeof(((%s)0)->%s)' % (tp.get_c_name('*'), fldname)
                 c_field.append(
                     '  { "%s", offsetof(%s, %s),\n' % (
                             fldname, tp.get_c_name(''), fldname) +
-                    '     %s   sizeof(((%s)0)->%s),\n' % (
-                            spaces, tp.get_c_name('*'), fldname) +
+                    '     %s   %s,\n' % (spaces, size) +
                     '     %s   _CFFI_OP(_CFFI_OP_NOOP, %s) },' % (
                             spaces, self._typesdict[fldtype]))
             self._lsts["field"].append('\n'.join(c_field))
@@ -495,7 +499,7 @@
     def _global_type(self, tp, global_name):
         if isinstance(tp, model.ArrayType) and tp.length == '...':
             actual_length = '_cffi_array_len(%s)' % (global_name,)
-            tp = model.ArrayType(tp.item, actual_length)
+            tp = tp.resolve_length(actual_length)
         return tp
 
     def _generate_cpy_variable_collecttype(self, tp, name):
diff --git a/new/test_recompiler.py b/new/test_recompiler.py
--- a/new/test_recompiler.py
+++ b/new/test_recompiler.py
@@ -252,3 +252,12 @@
     e = py.test.raises(ffi.error, "p.a")     # lazily build the fields and boom
     assert str(e.value) == ("struct foo_s field 'a' was declared in the "
                             "cdef to be 20 bytes, but is actually 24 bytes")
+
+def test_open_array_in_struct():
+    ffi = FFI()
+    ffi.cdef("struct foo_s { int b; int a[]; };")
+    verify(ffi, 'test_open_array_in_struct',
+           "struct foo_s { int b; int a[]; };")
+    assert ffi.sizeof("struct foo_s") == 4
+    p = ffi.new("struct foo_s *", [5, [10, 20, 30]])
+    assert p.a[2] == 30

From noreply at buildbot.pypy.org  Sat Apr 18 12:58:26 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 18 Apr 2015 12:58:26 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: improve readability of the generated
	file
Message-ID: <20150418105826.6C7281C03F7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1753:25edcc21fda2
Date: 2015-04-18 12:59 +0200
http://bitbucket.org/cffi/cffi/changeset/25edcc21fda2/

Log:	improve readability of the generated file

diff --git a/new/recompiler.py b/new/recompiler.py
--- a/new/recompiler.py
+++ b/new/recompiler.py
@@ -118,8 +118,8 @@
         #
         # the declaration of '_cffi_types'
         prnt('static void *_cffi_types[] = {')
-        for op in self.cffi_types:
-            prnt('  %s,' % (op.as_c_expr(),))
+        for i, op in enumerate(self.cffi_types):
+            prnt('/* %2d */ %s,' % (i, op.as_c_expr()))
         if not self.cffi_types:
             prnt('  0')
         prnt('};')

From noreply at buildbot.pypy.org  Sat Apr 18 13:27:05 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 18 Apr 2015 13:27:05 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: fixes
Message-ID: <20150418112705.E1B3E1C03F7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1754:6d8389513bdb
Date: 2015-04-18 13:27 +0200
http://bitbucket.org/cffi/cffi/changeset/6d8389513bdb/

Log:	fixes

diff --git a/cffi/model.py b/cffi/model.py
--- a/cffi/model.py
+++ b/cffi/model.py
@@ -454,6 +454,7 @@
         structname = '$%s' % name
     tp = StructType(structname, None, None, None)
     tp.force_the_name(name)
+    tp.origin = "unknown_type"
     return tp
 
 def unknown_ptr_type(name, structname=None):
diff --git a/new/recompiler.py b/new/recompiler.py
--- a/new/recompiler.py
+++ b/new/recompiler.py
@@ -150,6 +150,14 @@
                 prnt('};')
                 prnt()
         #
+        # check for a possible internal inconsistency: _cffi_struct_unions
+        # should have been generated with exactly self._struct_unions
+        lst = self._lsts["struct_union"]
+        for tp, i in self._struct_unions.items():
+            assert i < len(lst)
+            assert lst[i].startswith('  { "%s"' % tp.name)
+        assert len(lst) == len(self._struct_unions)
+        #
         # the declaration of '_cffi_type_context'
         prnt('static const struct _cffi_type_context_s _cffi_type_context = {')
         prnt('  _cffi_types,')
@@ -271,6 +279,10 @@
         type_index = self._typesdict[tp]
         self._lsts["typename"].append(
             '  { "%s", %d },' % (name, type_index))
+        if getattr(tp, "origin", None) == "unknown_type":
+            self._generate_cpy_struct_ctx(tp, tp.name)
+        elif isinstance(tp, model.NamedPointerType):
+            self._generate_cpy_struct_ctx(tp.totype, tp.totype.name)
 
     # ----------
     # function declarations
@@ -539,6 +551,7 @@
         self.cffi_types[index] = CffiOp(OP_POINTER, self._typesdict[tp.totype])
 
     _emit_bytecode_ConstPointerType = _emit_bytecode_PointerType
+    _emit_bytecode_NamedPointerType = _emit_bytecode_PointerType
 
     def _emit_bytecode_FunctionPtrType(self, tp, index):
         raw = tp.as_raw_function()
diff --git a/new/test_recompiler.py b/new/test_recompiler.py
--- a/new/test_recompiler.py
+++ b/new/test_recompiler.py
@@ -118,6 +118,16 @@
     lib = verify(ffi, 'test_verify_typedef', 'typedef int **foo_t;')
     assert ffi.sizeof("foo_t") == ffi.sizeof("void *")
 
+def test_verify_typedef_dotdotdot():
+    ffi = FFI()
+    ffi.cdef("typedef ... foo_t;")
+    verify(ffi, 'test_verify_typedef_dotdotdot', 'typedef int **foo_t;')
+
+def test_verify_typedef_star_dotdotdot():
+    ffi = FFI()
+    ffi.cdef("typedef ... *foo_t;")
+    verify(ffi, 'test_verify_typedef_star_dotdotdot', 'typedef int **foo_t;')
+
 def test_global_var_int():
     ffi = FFI()
     ffi.cdef("int a, b, c;")

From noreply at buildbot.pypy.org  Sat Apr 18 13:34:34 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 18 Apr 2015 13:34:34 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Generate more comments
Message-ID: <20150418113434.08D281C03F7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1755:3ccb4668ee1f
Date: 2015-04-18 13:35 +0200
http://bitbucket.org/cffi/cffi/changeset/3ccb4668ee1f/

Log:	Generate more comments

diff --git a/new/recompiler.py b/new/recompiler.py
--- a/new/recompiler.py
+++ b/new/recompiler.py
@@ -118,8 +118,12 @@
         #
         # the declaration of '_cffi_types'
         prnt('static void *_cffi_types[] = {')
+        typeindex2type = dict([(i, tp) for (tp, i) in self._typesdict.items()])
         for i, op in enumerate(self.cffi_types):
-            prnt('/* %2d */ %s,' % (i, op.as_c_expr()))
+            comment = ''
+            if i in typeindex2type:
+                comment = ' // ' + typeindex2type[i]._get_c_name()
+            prnt('/* %2d */ %s,%s' % (i, op.as_c_expr(), comment))
         if not self.cffi_types:
             prnt('  0')
         prnt('};')

From noreply at buildbot.pypy.org  Sat Apr 18 13:57:37 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 18 Apr 2015 13:57:37 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Fix for a crash found by running
 test_open_array_in_struct in a debug
Message-ID: <20150418115737.020A11C0484@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1756:c880c45d083e
Date: 2015-04-18 13:58 +0200
http://bitbucket.org/cffi/cffi/changeset/c880c45d083e/

Log:	Fix for a crash found by running test_open_array_in_struct in a
	debug version of python. Then fix another test to account for the
	fix.

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -2844,6 +2844,10 @@
         if (ctitem->ct_flags & CT_PRIMITIVE_CHAR)
             datasize *= 2;   /* forcefully add another character: a null */
 
+        if (ctitem->ct_flags & (CT_STRUCT | CT_UNION)) {
+            if (force_lazy_struct(ctitem) < 0)   /* for CT_WITH_VAR_ARRAY */
+                return NULL;
+        }
         if ((ctitem->ct_flags & CT_WITH_VAR_ARRAY) && init != Py_None) {
             Py_ssize_t optvarsize = datasize;
             if (convert_struct_from_object(NULL,ctitem, init, &optvarsize) < 0)
diff --git a/new/test_recompiler.py b/new/test_recompiler.py
--- a/new/test_recompiler.py
+++ b/new/test_recompiler.py
@@ -258,8 +258,8 @@
     verify(ffi, 'test_misdeclared_field_1',
            "struct foo_s { int a[6]; };")
     assert ffi.sizeof("struct foo_s") == 24  # found by the actual C code
-    p = ffi.new("struct foo_s *")
-    e = py.test.raises(ffi.error, "p.a")     # lazily build the fields and boom
+    # lazily build the fields and boom:
+    e = py.test.raises(ffi.error, ffi.new, "struct foo_s *")
     assert str(e.value) == ("struct foo_s field 'a' was declared in the "
                             "cdef to be 20 bytes, but is actually 24 bytes")
 

From noreply at buildbot.pypy.org  Sat Apr 18 17:09:18 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 18 Apr 2015 17:09:18 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: typeof(built-in method of lib object)
Message-ID: <20150418150918.071C21C03F7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1757:d683086b3771
Date: 2015-04-18 17:09 +0200
http://bitbucket.org/cffi/cffi/changeset/d683086b3771/

Log:	typeof(built-in method of lib object)

diff --git a/new/ffi_obj.c b/new/ffi_obj.c
--- a/new/ffi_obj.c
+++ b/new/ffi_obj.c
@@ -195,14 +195,16 @@
 "corresponding  object.\n"
 "It can also be used on 'cdata' instance to get its C type.");
 
+static PyObject *_cpyextfunc_type_index(PyObject *x);  /* forward */
+
 static PyObject *ffi_typeof(FFIObject *self, PyObject *arg)
 {
     PyObject *x = (PyObject *)_ffi_type(self, arg, ACCEPT_STRING|ACCEPT_CDATA);
     if (x != NULL) {
         Py_INCREF(x);
     }
-    else if (PyCFunction_Check(arg)) {
-        abort(); // XXX
+    else {
+        x = _cpyextfunc_type_index(arg);
     }
     return x;
 }
diff --git a/new/lib_obj.c b/new/lib_obj.c
--- a/new/lib_obj.c
+++ b/new/lib_obj.c
@@ -13,10 +13,11 @@
 
 struct CPyExtFunc_s {
     PyMethodDef md;
-    builder_c_t *types_builder;
     int type_index;
 };
 
+#define METH_CPYEXTFUNC   0x40000000
+
 struct LibObject_s {
     PyObject_HEAD
     builder_c_t *l_types_builder; /* same as the one on the ffi object */
@@ -26,6 +27,38 @@
 
 #define LibObject_Check(ob)  ((Py_TYPE(ob) == &Lib_Type))
 
+static PyObject *_cpyextfunc_type_index(PyObject *x)
+{
+    assert(PyErr_Occurred());
+
+    if (!PyCFunction_Check(x))
+        return NULL;
+    if (!LibObject_Check(PyCFunction_GET_SELF(x)))
+        return NULL;
+    if (!(PyCFunction_GET_FLAGS(x) & METH_CPYEXTFUNC))
+        return NULL;
+
+    PyErr_Clear();
+
+    LibObject *lib = (LibObject *)PyCFunction_GET_SELF(x);
+    struct CPyExtFunc_s *exf;
+    PyObject *tuple, *result;
+
+    exf = (struct CPyExtFunc_s *)(((PyCFunctionObject *)x) -> m_ml);
+    tuple = _realize_c_type_or_func(lib->l_types_builder,
+                                    lib->l_types_builder->ctx.types,
+                                    exf->type_index);
+    if (tuple == NULL)
+        return NULL;
+
+    /* 'tuple' is a tuple of length 1 containing the real CT_FUNCTIONPTR
+       object */
+    result = PyTuple_GetItem(tuple, 0);
+    Py_XINCREF(result);
+    Py_DECREF(tuple);
+    return result;
+}
+
 static void lib_dealloc(LibObject *lib)
 {
     Py_DECREF(lib->l_dict);
@@ -79,16 +112,15 @@
         goto no_memory;
 
     xfunc->md.ml_meth = (PyCFunction)g->address;
-    xfunc->md.ml_flags = flags;
+    xfunc->md.ml_flags = flags | METH_CPYEXTFUNC;
     xfunc->md.ml_name = g->name;
     /*xfunc->md.ml_doc = ... */
     if (xfunc->md.ml_name == NULL)
         goto no_memory;
 
-    xfunc->types_builder = lib->l_types_builder;
     xfunc->type_index = type_index;
 
-    return PyCFunction_NewEx(&xfunc->md, NULL, lib->l_libname);
+    return PyCFunction_NewEx(&xfunc->md, (PyObject *)lib, lib->l_libname);
 
  no_memory:
     PyErr_NoMemory();
diff --git a/new/test_recompiler.py b/new/test_recompiler.py
--- a/new/test_recompiler.py
+++ b/new/test_recompiler.py
@@ -271,3 +271,13 @@
     assert ffi.sizeof("struct foo_s") == 4
     p = ffi.new("struct foo_s *", [5, [10, 20, 30]])
     assert p.a[2] == 30
+
+def test_math_sin_type():
+    ffi = FFI()
+    ffi.cdef("double sin(double);")
+    lib = verify(ffi, 'test_math_sin_type', '#include ')
+    # 'lib.sin' is typed as a  object on lib
+    assert ffi.typeof(lib.sin).cname == "double(*)(double)"
+    # 'x' is another  object on lib, made very indirectly
+    x = type(lib).__dir__.__get__(lib)
+    py.test.raises(TypeError, ffi.typeof, x)

From noreply at buildbot.pypy.org  Sat Apr 18 17:33:22 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 18 Apr 2015 17:33:22 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Copy the bulk of test_verify here,
	for now.
Message-ID: <20150418153322.084121C0484@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1758:a57ecdda914b
Date: 2015-04-18 17:23 +0200
http://bitbucket.org/cffi/cffi/changeset/a57ecdda914b/

Log:	Copy the bulk of test_verify here, for now.

diff too long, truncating to 2000 out of 2122 lines

diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -319,7 +319,11 @@
     def _get_cached_btype(self, type):
         raise DeprecatedError
 
-    def verify(self, source='', tmpdir=None, **kwargs):
+    def verify(self, source='', **kwargs):
+        from recompiler import verify    # XXX must be in the current dir
+        return verify(self, 'verify', source, **kwargs)
+
+    def XXXverify(self, source='', tmpdir=None, **kwargs):
         """Verify that the current ffi signatures compile on this
         machine, and return a dynamic library object.  The dynamic
         library can be used to call functions and access global
diff --git a/new/test_verify1.py b/new/test_verify1.py
new file mode 100644
--- /dev/null
+++ b/new/test_verify1.py
@@ -0,0 +1,2101 @@
+import sys
+from cffi1 import FFI, VerificationError, model
+
+
+def test_missing_function(ffi=None):
+    # uses the FFI hacked above with '-Werror'
+    if ffi is None:
+        ffi = FFI()
+    ffi.cdef("void some_completely_unknown_function();")
+    try:
+        lib = ffi.verify()
+    except (VerificationError, OSError):
+        pass     # expected case: we get a VerificationError
+    else:
+        # but depending on compiler and loader details, maybe
+        # 'lib' could actually be imported but will fail if we
+        # actually try to call the unknown function...  Hard
+        # to test anything more.
+        pass
+
+def test_missing_function_import_error():
+    # uses the original FFI that just gives a warning during compilation
+    import cffi
+    test_missing_function(ffi=cffi.FFI())
+
+def test_simple_case():
+    ffi = FFI()
+    ffi.cdef("double sin(double x);")
+    lib = ffi.verify('#include ', libraries=lib_m)
+    assert lib.sin(1.23) == math.sin(1.23)
+
+def _Wconversion(cdef, source, **kargs):
+    if sys.platform == 'win32':
+        py.test.skip("needs GCC or Clang")
+    ffi = FFI()
+    ffi.cdef(cdef)
+    py.test.raises(VerificationError, ffi.verify, source, **kargs)
+    extra_compile_args_orig = extra_compile_args[:]
+    extra_compile_args.remove('-Wconversion')
+    try:
+        lib = ffi.verify(source, **kargs)
+    finally:
+        extra_compile_args[:] = extra_compile_args_orig
+    return lib
+
+def test_Wconversion_unsigned():
+    _Wconversion("unsigned foo(void);",
+                 "int foo(void) { return -1;}")
+
+def test_Wconversion_integer():
+    _Wconversion("short foo(void);",
+                 "long long foo(void) { return 1<", libraries=lib_m)
+    res = lib.sin(1.23)
+    assert res != math.sin(1.23)     # not exact, because of double->float
+    assert abs(res - math.sin(1.23)) < 1E-5
+
+def test_Wconversion_float2int():
+    _Wconversion("int sinf(float);",
+                 "#include ", libraries=lib_m)
+
+def test_Wconversion_double2int():
+    _Wconversion("int sin(double);",
+                 "#include ", libraries=lib_m)
+
+def test_rounding_1():
+    ffi = FFI()
+    ffi.cdef("double sinf(float x);")
+    lib = ffi.verify('#include ', libraries=lib_m)
+    res = lib.sinf(1.23)
+    assert res != math.sin(1.23)     # not exact, because of double->float
+    assert abs(res - math.sin(1.23)) < 1E-5
+
+def test_rounding_2():
+    ffi = FFI()
+    ffi.cdef("double sin(float x);")
+    lib = ffi.verify('#include ', libraries=lib_m)
+    res = lib.sin(1.23)
+    assert res != math.sin(1.23)     # not exact, because of double->float
+    assert abs(res - math.sin(1.23)) < 1E-5
+
+def test_strlen_exact():
+    ffi = FFI()
+    ffi.cdef("size_t strlen(const char *s);")
+    lib = ffi.verify("#include ")
+    assert lib.strlen(b"hi there!") == 9
+
+def test_strlen_approximate():
+    lib = _Wconversion("int strlen(char *s);",
+                       "#include ")
+    assert lib.strlen(b"hi there!") == 9
+
+def test_return_approximate():
+    for typename in ['short', 'int', 'long', 'long long']:
+        ffi = FFI()
+        ffi.cdef("%s foo(signed char x);" % typename)
+        lib = ffi.verify("signed char foo(signed char x) { return x;}")
+        assert lib.foo(-128) == -128
+        assert lib.foo(+127) == +127
+
+def test_strlen_array_of_char():
+    ffi = FFI()
+    ffi.cdef("size_t strlen(char[]);")
+    lib = ffi.verify("#include ")
+    assert lib.strlen(b"hello") == 5
+
+def test_longdouble():
+    ffi = FFI()
+    ffi.cdef("long double sinl(long double x);")
+    lib = ffi.verify('#include ', libraries=lib_m)
+    for input in [1.23,
+                  ffi.cast("double", 1.23),
+                  ffi.cast("long double", 1.23)]:
+        x = lib.sinl(input)
+        assert repr(x).startswith(" 0.1
+        # Check the particular results on Intel
+        import platform
+        if (platform.machine().startswith('i386') or
+            platform.machine().startswith('x86')):
+            assert abs(more_precise - 0.656769) < 0.001
+            assert abs(less_precise - 3.99091) < 0.001
+        else:
+            py.test.skip("don't know the very exact precision of 'long double'")
+
+
+all_primitive_types = model.PrimitiveType.ALL_PRIMITIVE_TYPES
+if sys.platform == 'win32':
+    all_primitive_types = all_primitive_types.copy()
+    del all_primitive_types['ssize_t']
+all_integer_types = sorted(tp for tp in all_primitive_types
+                           if all_primitive_types[tp] == 'i')
+all_float_types = sorted(tp for tp in all_primitive_types
+                            if all_primitive_types[tp] == 'f')
+
+def all_signed_integer_types(ffi):
+    return [x for x in all_integer_types if int(ffi.cast(x, -1)) < 0]
+
+def all_unsigned_integer_types(ffi):
+    return [x for x in all_integer_types if int(ffi.cast(x, -1)) > 0]
+
+
+def test_primitive_category():
+    for typename in all_primitive_types:
+        tp = model.PrimitiveType(typename)
+        C = tp.is_char_type()
+        F = tp.is_float_type()
+        I = tp.is_integer_type()
+        assert C == (typename in ('char', 'wchar_t'))
+        assert F == (typename in ('float', 'double', 'long double'))
+        assert I + F + C == 1      # one and only one of them is true
+
+def test_all_integer_and_float_types():
+    typenames = []
+    for typename in all_primitive_types:
+        if (all_primitive_types[typename] == 'c' or
+            typename == '_Bool' or typename == 'long double'):
+            pass
+        else:
+            typenames.append(typename)
+    #
+    ffi = FFI()
+    ffi.cdef('\n'.join(["%s foo_%s(%s);" % (tp, tp.replace(' ', '_'), tp)
+                       for tp in typenames]))
+    lib = ffi.verify('\n'.join(["%s foo_%s(%s x) { return (%s)(x+1); }" %
+                                (tp, tp.replace(' ', '_'), tp, tp)
+                                for tp in typenames]))
+    for typename in typenames:
+        foo = getattr(lib, 'foo_%s' % typename.replace(' ', '_'))
+        assert foo(42) == 43
+        if sys.version < '3':
+            assert foo(long(44)) == 45
+        assert foo(ffi.cast(typename, 46)) == 47
+        py.test.raises(TypeError, foo, ffi.NULL)
+        #
+        # check for overflow cases
+        if all_primitive_types[typename] == 'f':
+            continue
+        for value in [-2**80, -2**40, -2**20, -2**10, -2**5, -1,
+                      2**5, 2**10, 2**20, 2**40, 2**80]:
+            overflows = int(ffi.cast(typename, value)) != value
+            if overflows:
+                py.test.raises(OverflowError, foo, value)
+            else:
+                assert foo(value) == value + 1
+
+def test_var_signed_integer_types():
+    ffi = FFI()
+    lst = all_signed_integer_types(ffi)
+    csource = "\n".join(["%s somevar_%s;" % (tp, tp.replace(' ', '_'))
+                         for tp in lst])
+    ffi.cdef(csource)
+    lib = ffi.verify(csource)
+    for tp in lst:
+        varname = 'somevar_%s' % tp.replace(' ', '_')
+        sz = ffi.sizeof(tp)
+        max = (1 << (8*sz-1)) - 1
+        min = -(1 << (8*sz-1))
+        setattr(lib, varname, max)
+        assert getattr(lib, varname) == max
+        setattr(lib, varname, min)
+        assert getattr(lib, varname) == min
+        py.test.raises(OverflowError, setattr, lib, varname, max+1)
+        py.test.raises(OverflowError, setattr, lib, varname, min-1)
+
+def test_var_unsigned_integer_types():
+    ffi = FFI()
+    lst = all_unsigned_integer_types(ffi)
+    csource = "\n".join(["%s somevar_%s;" % (tp, tp.replace(' ', '_'))
+                         for tp in lst])
+    ffi.cdef(csource)
+    lib = ffi.verify(csource)
+    for tp in lst:
+        varname = 'somevar_%s' % tp.replace(' ', '_')
+        sz = ffi.sizeof(tp)
+        if tp != '_Bool':
+            max = (1 << (8*sz)) - 1
+        else:
+            max = 1
+        setattr(lib, varname, max)
+        assert getattr(lib, varname) == max
+        setattr(lib, varname, 0)
+        assert getattr(lib, varname) == 0
+        py.test.raises(OverflowError, setattr, lib, varname, max+1)
+        py.test.raises(OverflowError, setattr, lib, varname, -1)
+
+def test_fn_signed_integer_types():
+    ffi = FFI()
+    lst = all_signed_integer_types(ffi)
+    cdefsrc = "\n".join(["%s somefn_%s(%s);" % (tp, tp.replace(' ', '_'), tp)
+                         for tp in lst])
+    ffi.cdef(cdefsrc)
+    verifysrc = "\n".join(["%s somefn_%s(%s x) { return x; }" %
+                           (tp, tp.replace(' ', '_'), tp) for tp in lst])
+    lib = ffi.verify(verifysrc)
+    for tp in lst:
+        fnname = 'somefn_%s' % tp.replace(' ', '_')
+        sz = ffi.sizeof(tp)
+        max = (1 << (8*sz-1)) - 1
+        min = -(1 << (8*sz-1))
+        fn = getattr(lib, fnname)
+        assert fn(max) == max
+        assert fn(min) == min
+        py.test.raises(OverflowError, fn, max + 1)
+        py.test.raises(OverflowError, fn, min - 1)
+
+def test_fn_unsigned_integer_types():
+    ffi = FFI()
+    lst = all_unsigned_integer_types(ffi)
+    cdefsrc = "\n".join(["%s somefn_%s(%s);" % (tp, tp.replace(' ', '_'), tp)
+                         for tp in lst])
+    ffi.cdef(cdefsrc)
+    verifysrc = "\n".join(["%s somefn_%s(%s x) { return x; }" %
+                           (tp, tp.replace(' ', '_'), tp) for tp in lst])
+    lib = ffi.verify(verifysrc)
+    for tp in lst:
+        fnname = 'somefn_%s' % tp.replace(' ', '_')
+        sz = ffi.sizeof(tp)
+        if tp != '_Bool':
+            max = (1 << (8*sz)) - 1
+        else:
+            max = 1
+        fn = getattr(lib, fnname)
+        assert fn(max) == max
+        assert fn(0) == 0
+        py.test.raises(OverflowError, fn, max + 1)
+        py.test.raises(OverflowError, fn, -1)
+
+def test_char_type():
+    ffi = FFI()
+    ffi.cdef("char foo(char);")
+    lib = ffi.verify("char foo(char x) { return ++x; }")
+    assert lib.foo(b"A") == b"B"
+    py.test.raises(TypeError, lib.foo, b"bar")
+    py.test.raises(TypeError, lib.foo, "bar")
+
+def test_wchar_type():
+    ffi = FFI()
+    if ffi.sizeof('wchar_t') == 2:
+        uniexample1 = u+'\u1234'
+        uniexample2 = u+'\u1235'
+    else:
+        uniexample1 = u+'\U00012345'
+        uniexample2 = u+'\U00012346'
+    #
+    ffi.cdef("wchar_t foo(wchar_t);")
+    lib = ffi.verify("wchar_t foo(wchar_t x) { return x+1; }")
+    assert lib.foo(uniexample1) == uniexample2
+
+def test_no_argument():
+    ffi = FFI()
+    ffi.cdef("int foo(void);")
+    lib = ffi.verify("int foo(void) { return 42; }")
+    assert lib.foo() == 42
+
+def test_two_arguments():
+    ffi = FFI()
+    ffi.cdef("int foo(int, int);")
+    lib = ffi.verify("int foo(int a, int b) { return a - b; }")
+    assert lib.foo(40, -2) == 42
+
+def test_macro():
+    ffi = FFI()
+    ffi.cdef("int foo(int, int);")
+    lib = ffi.verify("#define foo(a, b) ((a) * (b))")
+    assert lib.foo(-6, -7) == 42
+
+def test_ptr():
+    ffi = FFI()
+    ffi.cdef("int *foo(int *);")
+    lib = ffi.verify("int *foo(int *a) { return a; }")
+    assert lib.foo(ffi.NULL) == ffi.NULL
+    p = ffi.new("int *", 42)
+    q = ffi.new("int *", 42)
+    assert lib.foo(p) == p
+    assert lib.foo(q) != p
+
+def test_bogus_ptr():
+    ffi = FFI()
+    ffi.cdef("int *foo(int *);")
+    lib = ffi.verify("int *foo(int *a) { return a; }")
+    py.test.raises(TypeError, lib.foo, ffi.new("short *", 42))
+
+
+def test_verify_typedefs():
+    py.test.skip("ignored so far")
+    types = ['signed char', 'unsigned char', 'int', 'long']
+    for cdefed in types:
+        for real in types:
+            ffi = FFI()
+            ffi.cdef("typedef %s foo_t;" % cdefed)
+            if cdefed == real:
+                ffi.verify("typedef %s foo_t;" % real)
+            else:
+                py.test.raises(VerificationError, ffi.verify,
+                               "typedef %s foo_t;" % real)
+
+def test_nondecl_struct():
+    ffi = FFI()
+    ffi.cdef("typedef struct foo_s foo_t; int bar(foo_t *);")
+    lib = ffi.verify("typedef struct foo_s foo_t;\n"
+                     "int bar(foo_t *f) { (void)f; return 42; }\n")
+    assert lib.bar(ffi.NULL) == 42
+
+def test_ffi_full_struct():
+    ffi = FFI()
+    ffi.cdef("struct foo_s { char x; int y; long *z; };")
+    ffi.verify("struct foo_s { char x; int y; long *z; };")
+    #
+    if sys.platform != 'win32':  # XXX fixme: only gives warnings
+        py.test.raises(VerificationError, ffi.verify,
+            "struct foo_s { char x; int y; int *z; };")
+    #
+    py.test.raises(VerificationError, ffi.verify,
+        "struct foo_s { int y; long *z; };")
+    #
+    e = py.test.raises(VerificationError, ffi.verify,
+        "struct foo_s { int y; char x; long *z; };")
+    assert str(e.value) == (
+        "struct foo_s: wrong offset for field 'x'"
+        " (we have 0, but C compiler says 4)")
+    #
+    e = py.test.raises(VerificationError, ffi.verify,
+        "struct foo_s { char x; int y; long *z; char extra; };")
+    assert str(e.value) == (
+        "struct foo_s: wrong total size"
+        " (we have %d, but C compiler says %d)" % (
+            ffi.sizeof("struct foo_s"),
+            ffi.sizeof("struct foo_s") + ffi.sizeof("long*")))
+    #
+    # a corner case that we cannot really detect, but where it has no
+    # bad consequences: the size is the same, but there is an extra field
+    # that replaces what is just padding in our declaration above
+    ffi.verify("struct foo_s { char x, extra; int y; long *z; };")
+    #
+    e = py.test.raises(VerificationError, ffi.verify,
+        "struct foo_s { char x; short pad; short y; long *z; };")
+    assert str(e.value) == (
+        "struct foo_s: wrong size for field 'y'"
+        " (we have 4, but C compiler says 2)")
+
+def test_ffi_nonfull_struct():
+    ffi = FFI()
+    ffi.cdef("""
+    struct foo_s {
+       int x;
+       ...;
+    };
+    """)
+    py.test.raises(VerificationMissing, ffi.sizeof, 'struct foo_s')
+    py.test.raises(VerificationMissing, ffi.offsetof, 'struct foo_s', 'x')
+    py.test.raises(VerificationMissing, ffi.new, 'struct foo_s *')
+    ffi.verify("""
+    struct foo_s {
+       int a, b, x, c, d, e;
+    };
+    """)
+    assert ffi.sizeof('struct foo_s') == 6 * ffi.sizeof('int')
+    assert ffi.offsetof('struct foo_s', 'x') == 2 * ffi.sizeof('int')
+
+def test_ffi_nonfull_alignment():
+    ffi = FFI()
+    ffi.cdef("struct foo_s { char x; ...; };")
+    ffi.verify("struct foo_s { int a, b; char x; };")
+    assert ffi.sizeof('struct foo_s') == 3 * ffi.sizeof('int')
+    assert ffi.alignof('struct foo_s') == ffi.sizeof('int')
+
+def _check_field_match(typename, real, expect_mismatch):
+    ffi = FFI()
+    testing_by_size = (expect_mismatch == 'by_size')
+    if testing_by_size:
+        expect_mismatch = ffi.sizeof(typename) != ffi.sizeof(real)
+    ffi.cdef("struct foo_s { %s x; ...; };" % typename)
+    try:
+        ffi.verify("struct foo_s { %s x; };" % real)
+    except VerificationError:
+        if not expect_mismatch:
+            if testing_by_size and typename != real:
+                print("ignoring mismatch between %s* and %s* even though "
+                      "they have the same size" % (typename, real))
+                return
+            raise AssertionError("unexpected mismatch: %s should be accepted "
+                                 "as equal to %s" % (typename, real))
+    else:
+        if expect_mismatch:
+            raise AssertionError("mismatch not detected: "
+                                 "%s != %s" % (typename, real))
+
+def test_struct_bad_sized_integer():
+    for typename in ['int8_t', 'int16_t', 'int32_t', 'int64_t']:
+        for real in ['int8_t', 'int16_t', 'int32_t', 'int64_t']:
+            _check_field_match(typename, real, "by_size")
+
+def test_struct_bad_sized_float():
+    for typename in all_float_types:
+        for real in all_float_types:
+            _check_field_match(typename, real, "by_size")
+
+def test_struct_signedness_ignored():
+    _check_field_match("int", "unsigned int", expect_mismatch=False)
+    _check_field_match("unsigned short", "signed short", expect_mismatch=False)
+
+def test_struct_float_vs_int():
+    if sys.platform == 'win32':
+        py.test.skip("XXX fixme: only gives warnings")
+    ffi = FFI()
+    for typename in all_signed_integer_types(ffi):
+        for real in all_float_types:
+            _check_field_match(typename, real, expect_mismatch=True)
+    for typename in all_float_types:
+        for real in all_signed_integer_types(ffi):
+            _check_field_match(typename, real, expect_mismatch=True)
+
+def test_struct_array_field():
+    ffi = FFI()
+    ffi.cdef("struct foo_s { int a[17]; ...; };")
+    ffi.verify("struct foo_s { int x; int a[17]; int y; };")
+    assert ffi.sizeof('struct foo_s') == 19 * ffi.sizeof('int')
+    s = ffi.new("struct foo_s *")
+    assert ffi.sizeof(s.a) == 17 * ffi.sizeof('int')
+
+def test_struct_array_no_length():
+    ffi = FFI()
+    ffi.cdef("struct foo_s { int a[]; int y; ...; };\n"
+             "int bar(struct foo_s *);\n")
+    lib = ffi.verify("struct foo_s { int x; int a[17]; int y; };\n"
+                     "int bar(struct foo_s *f) { return f->a[14]; }\n")
+    assert ffi.sizeof('struct foo_s') == 19 * ffi.sizeof('int')
+    s = ffi.new("struct foo_s *")
+    assert ffi.typeof(s.a) is ffi.typeof('int *')   # because no length
+    s.a[14] = 4242
+    assert lib.bar(s) == 4242
+    # with no declared length, out-of-bound accesses are not detected
+    s.a[17] = -521
+    assert s.y == s.a[17] == -521
+    #
+    s = ffi.new("struct foo_s *", {'a': list(range(17))})
+    assert s.a[16] == 16
+    # overflows at construction time not detected either
+    s = ffi.new("struct foo_s *", {'a': list(range(18))})
+    assert s.y == s.a[17] == 17
+
+def test_struct_array_guess_length():
+    ffi = FFI()
+    ffi.cdef("struct foo_s { int a[...]; };")
+    ffi.verify("struct foo_s { int x; int a[17]; int y; };")
+    assert ffi.sizeof('struct foo_s') == 19 * ffi.sizeof('int')
+    s = ffi.new("struct foo_s *")
+    assert ffi.sizeof(s.a) == 17 * ffi.sizeof('int')
+    py.test.raises(IndexError, 's.a[17]')
+
+def test_struct_array_c99_1():
+    if sys.platform == 'win32':
+        py.test.skip("requires C99")
+    ffi = FFI()
+    ffi.cdef("struct foo_s { int x; int a[]; };")
+    ffi.verify("struct foo_s { int x; int a[]; };")
+    assert ffi.sizeof('struct foo_s') == 1 * ffi.sizeof('int')
+    s = ffi.new("struct foo_s *", [424242, 4])
+    assert ffi.sizeof(s[0]) == 1 * ffi.sizeof('int')   # the same in C
+    assert s.a[3] == 0
+    s = ffi.new("struct foo_s *", [424242, [-40, -30, -20, -10]])
+    assert ffi.sizeof(s[0]) == 1 * ffi.sizeof('int')
+    assert s.a[3] == -10
+    s = ffi.new("struct foo_s *")
+    assert ffi.sizeof(s[0]) == 1 * ffi.sizeof('int')
+    s = ffi.new("struct foo_s *", [424242])
+    assert ffi.sizeof(s[0]) == 1 * ffi.sizeof('int')
+
+def test_struct_array_c99_2():
+    if sys.platform == 'win32':
+        py.test.skip("requires C99")
+    ffi = FFI()
+    ffi.cdef("struct foo_s { int x; int a[]; ...; };")
+    ffi.verify("struct foo_s { int x, y; int a[]; };")
+    assert ffi.sizeof('struct foo_s') == 2 * ffi.sizeof('int')
+    s = ffi.new("struct foo_s *", [424242, 4])
+    assert ffi.sizeof(s[0]) == 2 * ffi.sizeof('int')
+    assert s.a[3] == 0
+    s = ffi.new("struct foo_s *", [424242, [-40, -30, -20, -10]])
+    assert ffi.sizeof(s[0]) == 2 * ffi.sizeof('int')
+    assert s.a[3] == -10
+    s = ffi.new("struct foo_s *")
+    assert ffi.sizeof(s[0]) == 2 * ffi.sizeof('int')
+    s = ffi.new("struct foo_s *", [424242])
+    assert ffi.sizeof(s[0]) == 2 * ffi.sizeof('int')
+
+def test_struct_ptr_to_array_field():
+    ffi = FFI()
+    ffi.cdef("struct foo_s { int (*a)[17]; ...; }; struct bar_s { ...; };")
+    ffi.verify("struct foo_s { int x; int (*a)[17]; int y; };\n"
+               "struct bar_s { int x; int *a; int y; };")
+    assert ffi.sizeof('struct foo_s') == ffi.sizeof("struct bar_s")
+    s = ffi.new("struct foo_s *")
+    assert ffi.sizeof(s.a) == ffi.sizeof('int(*)[17]') == ffi.sizeof("int *")
+
+def test_struct_with_bitfield_exact():
+    ffi = FFI()
+    ffi.cdef("struct foo_s { int a:2, b:3; };")
+    ffi.verify("struct foo_s { int a:2, b:3; };")
+    s = ffi.new("struct foo_s *")
+    s.b = 3
+    py.test.raises(OverflowError, "s.b = 4")
+    assert s.b == 3
+
+def test_struct_with_bitfield_enum():
+    ffi = FFI()
+    code = """
+        typedef enum { AA, BB, CC } foo_e;
+        typedef struct { foo_e f:2; } foo_s;
+    """
+    ffi.cdef(code)
+    ffi.verify(code)
+    s = ffi.new("foo_s *")
+    s.f = 2
+    assert s.f == 2
+
+def test_unsupported_struct_with_bitfield_ellipsis():
+    ffi = FFI()
+    py.test.raises(NotImplementedError, ffi.cdef,
+                   "struct foo_s { int a:2, b:3; ...; };")
+
+def test_global_constants():
+    ffi = FFI()
+    # use 'static const int', as generally documented, although in this
+    # case the 'static' is completely ignored.
+    ffi.cdef("static const int AA, BB, CC, DD;")
+    lib = ffi.verify("#define AA 42\n"
+                     "#define BB (-43)\n"
+                     "#define CC (22*2)\n"
+                     "#define DD ((unsigned int)142)\n")
+    assert lib.AA == 42
+    assert lib.BB == -43
+    assert lib.CC == 44
+    assert lib.DD == 142
+
+def test_global_const_int_size():
+    # integer constants: ignore the declared type, always just use the value
+    for value in [-2**63, -2**31, -2**15,
+                  2**15-1, 2**15, 2**31-1, 2**31, 2**32-1, 2**32,
+                  2**63-1, 2**63, 2**64-1]:
+        ffi = FFI()
+        if value == int(ffi.cast("long long", value)):
+            if value < 0:
+                vstr = '(-%dLL-1)' % (~value,)
+            else:
+                vstr = '%dLL' % value
+        elif value == int(ffi.cast("unsigned long long", value)):
+            vstr = '%dULL' % value
+        else:
+            raise AssertionError(value)
+        ffi.cdef("static const unsigned short AA;")
+        lib = ffi.verify("#define AA %s\n" % vstr)
+        assert lib.AA == value
+        assert type(lib.AA) is type(int(lib.AA))
+
+def test_global_constants_non_int():
+    ffi = FFI()
+    ffi.cdef("static char *const PP;")
+    lib = ffi.verify('static char *const PP = "testing!";\n')
+    assert ffi.typeof(lib.PP) == ffi.typeof("char *")
+    assert ffi.string(lib.PP) == b"testing!"
+
+def test_nonfull_enum():
+    ffi = FFI()
+    ffi.cdef("enum ee { EE1, EE2, EE3, ... \n \t };")
+    py.test.raises(VerificationMissing, ffi.cast, 'enum ee', 'EE2')
+    ffi.verify("enum ee { EE1=10, EE2, EE3=-10, EE4 };")
+    assert ffi.string(ffi.cast('enum ee', 11)) == "EE2"
+    assert ffi.string(ffi.cast('enum ee', -10)) == "EE3"
+    #
+    # try again
+    ffi.verify("enum ee { EE1=10, EE2, EE3=-10, EE4 };")
+    assert ffi.string(ffi.cast('enum ee', 11)) == "EE2"
+
+def test_full_enum():
+    ffi = FFI()
+    ffi.cdef("enum ee { EE1, EE2, EE3 };")
+    ffi.verify("enum ee { EE1, EE2, EE3 };")
+    py.test.raises(VerificationError, ffi.verify, "enum ee { EE1, EE2 };")
+    e = py.test.raises(VerificationError, ffi.verify,
+                       "enum ee { EE1, EE3, EE2 };")
+    assert str(e.value) == 'enum ee: EE2 has the real value 2, not 1'
+    # extra items cannot be seen and have no bad consequence anyway
+    lib = ffi.verify("enum ee { EE1, EE2, EE3, EE4 };")
+    assert lib.EE3 == 2
+
+def test_enum_usage():
+    ffi = FFI()
+    ffi.cdef("enum ee { EE1,EE2 }; typedef struct { enum ee x; } *sp;")
+    lib = ffi.verify("enum ee { EE1,EE2 }; typedef struct { enum ee x; } *sp;")
+    assert lib.EE2 == 1
+    s = ffi.new("sp", [lib.EE2])
+    assert s.x == 1
+    s.x = 17
+    assert s.x == 17
+
+def test_anonymous_enum():
+    ffi = FFI()
+    ffi.cdef("enum { EE1 }; enum { EE2, EE3 };")
+    lib = ffi.verify("enum { EE1 }; enum { EE2, EE3 };")
+    assert lib.EE1 == 0
+    assert lib.EE2 == 0
+    assert lib.EE3 == 1
+
+def test_nonfull_anonymous_enum():
+    ffi = FFI()
+    ffi.cdef("enum { EE1, ... }; enum { EE3, ... };")
+    lib = ffi.verify("enum { EE2, EE1 }; enum { EE3 };")
+    assert lib.EE1 == 1
+    assert lib.EE3 == 0
+
+def test_nonfull_enum_syntax2():
+    ffi = FFI()
+    ffi.cdef("enum ee { EE1, EE2=\t..., EE3 };")
+    py.test.raises(VerificationMissing, ffi.cast, 'enum ee', 'EE1')
+    ffi.verify("enum ee { EE1=10, EE2, EE3=-10, EE4 };")
+    assert ffi.string(ffi.cast('enum ee', 11)) == 'EE2'
+    assert ffi.string(ffi.cast('enum ee', -10)) == 'EE3'
+    #
+    ffi = FFI()
+    ffi.cdef("enum ee { EE1, EE2=\t... };")
+    py.test.raises(VerificationMissing, ffi.cast, 'enum ee', 'EE1')
+    ffi.verify("enum ee { EE1=10, EE2, EE3=-10, EE4 };")
+    assert ffi.string(ffi.cast('enum ee', 11)) == 'EE2'
+    #
+    ffi = FFI()
+    ffi.cdef("enum ee2 { EE4=..., EE5=..., ... };")
+    ffi.verify("enum ee2 { EE4=-1234-5, EE5 }; ")
+    assert ffi.string(ffi.cast('enum ee2', -1239)) == 'EE4'
+    assert ffi.string(ffi.cast('enum ee2', -1238)) == 'EE5'
+
+def test_get_set_errno():
+    ffi = FFI()
+    ffi.cdef("int foo(int);")
+    lib = ffi.verify("""
+        static int foo(int x)
+        {
+            errno += 1;
+            return x * 7;
+        }
+    """)
+    ffi.errno = 15
+    assert lib.foo(6) == 42
+    assert ffi.errno == 16
+
+def test_define_int():
+    ffi = FFI()
+    ffi.cdef("#define FOO ...\n"
+             "\t#\tdefine\tBAR\t...\t\n"
+             "#define BAZ ...\n")
+    lib = ffi.verify("#define FOO 42\n"
+                     "#define BAR (-44)\n"
+                     "#define BAZ 0xffffffffffffffffULL\n")
+    assert lib.FOO == 42
+    assert lib.BAR == -44
+    assert lib.BAZ == 0xffffffffffffffff
+
+def test_access_variable():
+    ffi = FFI()
+    ffi.cdef("int foo(void);\n"
+             "int somenumber;")
+    lib = ffi.verify("""
+        static int somenumber = 2;
+        static int foo(void) {
+            return somenumber * 7;
+        }
+    """)
+    assert lib.somenumber == 2
+    assert lib.foo() == 14
+    lib.somenumber = -6
+    assert lib.foo() == -42
+    assert lib.somenumber == -6
+    lib.somenumber = 2   # reset for the next run, if any
+
+def test_access_address_of_variable():
+    # access the address of 'somenumber': need a trick
+    ffi = FFI()
+    ffi.cdef("int somenumber; static int *const somenumberptr;")
+    lib = ffi.verify("""
+        static int somenumber = 2;
+        #define somenumberptr (&somenumber)
+    """)
+    assert lib.somenumber == 2
+    lib.somenumberptr[0] = 42
+    assert lib.somenumber == 42
+    lib.somenumber = 2    # reset for the next run, if any
+
+def test_access_array_variable(length=5):
+    ffi = FFI()
+    ffi.cdef("int foo(int);\n"
+             "int somenumber[%s];" % (length,))
+    lib = ffi.verify("""
+        static int somenumber[] = {2, 2, 3, 4, 5};
+        static int foo(int i) {
+            return somenumber[i] * 7;
+        }
+    """)
+    if length == '':
+        # a global variable of an unknown array length is implicitly
+        # transformed into a global pointer variable, because we can only
+        # work with array instances whose length we know.  using a pointer
+        # instead of an array gives the correct effects.
+        assert repr(lib.somenumber).startswith("x * 7; }
+    """)
+    f = ffi.new("foo_t *")
+    f.x = 6
+    assert lib.foo(f) == 42
+
+def test_unknown_type():
+    ffi = FFI()
+    ffi.cdef("""
+        typedef ... token_t;
+        int foo(token_t *);
+        #define TOKEN_SIZE ...
+    """)
+    lib = ffi.verify("""
+        typedef float token_t;
+        static int foo(token_t *tk) {
+            if (!tk)
+                return -42;
+            *tk += 1.601f;
+            return (int)*tk;
+        }
+        #define TOKEN_SIZE sizeof(token_t)
+    """)
+    # we cannot let ffi.new("token_t *") work, because we don't know ahead of
+    # time if it's ok to ask 'sizeof(token_t)' in the C code or not.
+    # See test_unknown_type_2.  Workaround.
+    tkmem = ffi.new("char[]", lib.TOKEN_SIZE)    # zero-initialized
+    tk = ffi.cast("token_t *", tkmem)
+    results = [lib.foo(tk) for i in range(6)]
+    assert results == [1, 3, 4, 6, 8, 9]
+    assert lib.foo(ffi.NULL) == -42
+
+def test_unknown_type_2():
+    ffi = FFI()
+    ffi.cdef("typedef ... token_t;")
+    lib = ffi.verify("typedef struct token_s token_t;")
+    # assert did not crash, even though 'sizeof(token_t)' is not valid in C.
+
+def test_unknown_type_3():
+    ffi = FFI()
+    ffi.cdef("""
+        typedef ... *token_p;
+        token_p foo(token_p);
+    """)
+    lib = ffi.verify("""
+        typedef struct _token_s *token_p;
+        token_p foo(token_p arg) {
+            if (arg)
+                return (token_p)0x12347;
+            else
+                return (token_p)0x12345;
+        }
+    """)
+    p = lib.foo(ffi.NULL)
+    assert int(ffi.cast("intptr_t", p)) == 0x12345
+    q = lib.foo(p)
+    assert int(ffi.cast("intptr_t", q)) == 0x12347
+
+def test_varargs():
+    ffi = FFI()
+    ffi.cdef("int foo(int x, ...);")
+    lib = ffi.verify("""
+        int foo(int x, ...) {
+            va_list vargs;
+            va_start(vargs, x);
+            x -= va_arg(vargs, int);
+            x -= va_arg(vargs, int);
+            va_end(vargs);
+            return x;
+        }
+    """)
+    assert lib.foo(50, ffi.cast("int", 5), ffi.cast("int", 3)) == 42
+
+def test_varargs_exact():
+    if sys.platform == 'win32':
+        py.test.skip("XXX fixme: only gives warnings")
+    ffi = FFI()
+    ffi.cdef("int foo(int x, ...);")
+    py.test.raises(VerificationError, ffi.verify, """
+        int foo(long long x, ...) {
+            return x;
+        }
+    """)
+
+def test_varargs_struct():
+    ffi = FFI()
+    ffi.cdef("struct foo_s { char a; int b; }; int foo(int x, ...);")
+    lib = ffi.verify("""
+        struct foo_s {
+            char a; int b;
+        };
+        int foo(int x, ...) {
+            va_list vargs;
+            struct foo_s s;
+            va_start(vargs, x);
+            s = va_arg(vargs, struct foo_s);
+            va_end(vargs);
+            return s.a - s.b;
+        }
+    """)
+    s = ffi.new("struct foo_s *", [b'B', 1])
+    assert lib.foo(50, s[0]) == ord('A')
+
+def test_autofilled_struct_as_argument():
+    ffi = FFI()
+    ffi.cdef("struct foo_s { long a; double b; ...; };\n"
+             "int foo(struct foo_s);")
+    lib = ffi.verify("""
+        struct foo_s {
+            double b;
+            long a;
+        };
+        int foo(struct foo_s s) {
+            return (int)s.a - (int)s.b;
+        }
+    """)
+    s = ffi.new("struct foo_s *", [100, 1])
+    assert lib.foo(s[0]) == 99
+    assert lib.foo([100, 1]) == 99
+
+def test_autofilled_struct_as_argument_dynamic():
+    ffi = FFI()
+    ffi.cdef("struct foo_s { long a; ...; };\n"
+             "int (*foo)(struct foo_s);")
+    e = py.test.raises(TypeError, ffi.verify, """
+        struct foo_s {
+            double b;
+            long a;
+        };
+        int foo1(struct foo_s s) {
+            return (int)s.a - (int)s.b;
+        }
+        int (*foo)(struct foo_s s) = &foo1;
+    """)
+    msg ='cannot pass as an argument a struct that was completed with verify()'
+    assert msg in str(e.value)
+
+def test_func_returns_struct():
+    ffi = FFI()
+    ffi.cdef("""
+        struct foo_s { int aa, bb; };
+        struct foo_s foo(int a, int b);
+    """)
+    lib = ffi.verify("""
+        struct foo_s { int aa, bb; };
+        struct foo_s foo(int a, int b) {
+            struct foo_s r;
+            r.aa = a*a;
+            r.bb = b*b;
+            return r;
+        }
+    """)
+    s = lib.foo(6, 7)
+    assert repr(s) == ""
+    assert s.aa == 36
+    assert s.bb == 49
+
+def test_func_as_funcptr():
+    ffi = FFI()
+    ffi.cdef("int *(*const fooptr)(void);")
+    lib = ffi.verify("""
+        int *foo(void) {
+            return (int*)"foobar";
+        }
+        int *(*fooptr)(void) = foo;
+    """)
+    foochar = ffi.cast("char *(*)(void)", lib.fooptr)
+    s = foochar()
+    assert ffi.string(s) == b"foobar"
+
+def test_funcptr_as_argument():
+    ffi = FFI()
+    ffi.cdef("""
+        void qsort(void *base, size_t nel, size_t width,
+            int (*compar)(const void *, const void *));
+    """)
+    ffi.verify("#include ")
+
+def test_func_as_argument():
+    ffi = FFI()
+    ffi.cdef("""
+        void qsort(void *base, size_t nel, size_t width,
+            int compar(const void *, const void *));
+    """)
+    ffi.verify("#include ")
+
+def test_array_as_argument():
+    ffi = FFI()
+    ffi.cdef("""
+        size_t strlen(char string[]);
+    """)
+    ffi.verify("#include ")
+
+def test_enum_as_argument():
+    ffi = FFI()
+    ffi.cdef("""
+        enum foo_e { AA, BB, ... };
+        int foo_func(enum foo_e);
+    """)
+    lib = ffi.verify("""
+        enum foo_e { AA, CC, BB };
+        int foo_func(enum foo_e e) { return (int)e; }
+    """)
+    assert lib.foo_func(lib.BB) == 2
+    py.test.raises(TypeError, lib.foo_func, "BB")
+
+def test_enum_as_function_result():
+    ffi = FFI()
+    ffi.cdef("""
+        enum foo_e { AA, BB, ... };
+        enum foo_e foo_func(int x);
+    """)
+    lib = ffi.verify("""
+        enum foo_e { AA, CC, BB };
+        enum foo_e foo_func(int x) { return (enum foo_e)x; }
+    """)
+    assert lib.foo_func(lib.BB) == lib.BB == 2
+
+def test_enum_values():
+    ffi = FFI()
+    ffi.cdef("enum enum1_e { AA, BB };")
+    lib = ffi.verify("enum enum1_e { AA, BB };")
+    assert lib.AA == 0
+    assert lib.BB == 1
+    assert ffi.string(ffi.cast("enum enum1_e", 1)) == 'BB'
+
+def test_typedef_complete_enum():
+    ffi = FFI()
+    ffi.cdef("typedef enum { AA, BB } enum1_t;")
+    lib = ffi.verify("typedef enum { AA, BB } enum1_t;")
+    assert ffi.string(ffi.cast("enum1_t", 1)) == 'BB'
+    assert lib.AA == 0
+    assert lib.BB == 1
+
+def test_typedef_broken_complete_enum():
+    ffi = FFI()
+    ffi.cdef("typedef enum { AA, BB } enum1_t;")
+    py.test.raises(VerificationError, ffi.verify,
+                   "typedef enum { AA, CC, BB } enum1_t;")
+
+def test_typedef_incomplete_enum():
+    ffi = FFI()
+    ffi.cdef("typedef enum { AA, BB, ... } enum1_t;")
+    lib = ffi.verify("typedef enum { AA, CC, BB } enum1_t;")
+    assert ffi.string(ffi.cast("enum1_t", 1)) == '1'
+    assert ffi.string(ffi.cast("enum1_t", 2)) == 'BB'
+    assert lib.AA == 0
+    assert lib.BB == 2
+
+def test_typedef_enum_as_argument():
+    ffi = FFI()
+    ffi.cdef("""
+        typedef enum { AA, BB, ... } foo_t;
+        int foo_func(foo_t);
+    """)
+    lib = ffi.verify("""
+        typedef enum { AA, CC, BB } foo_t;
+        int foo_func(foo_t e) { return (int)e; }
+    """)
+    assert lib.foo_func(lib.BB) == lib.BB == 2
+    py.test.raises(TypeError, lib.foo_func, "BB")
+
+def test_typedef_enum_as_function_result():
+    ffi = FFI()
+    ffi.cdef("""
+        typedef enum { AA, BB, ... } foo_t;
+        foo_t foo_func(int x);
+    """)
+    lib = ffi.verify("""
+        typedef enum { AA, CC, BB } foo_t;
+        foo_t foo_func(int x) { return (foo_t)x; }
+    """)
+    assert lib.foo_func(lib.BB) == lib.BB == 2
+
+def test_function_typedef():
+    ffi = FFI()
+    ffi.cdef("""
+        typedef double func_t(double);
+        func_t sin;
+    """)
+    lib = ffi.verify('#include ', libraries=lib_m)
+    assert lib.sin(1.23) == math.sin(1.23)
+
+def test_callback_calling_convention():
+    py.test.skip("later")
+    if sys.platform != 'win32':
+        py.test.skip("Windows only")
+    ffi = FFI()
+    ffi.cdef("""
+        int call1(int(*__cdecl cb)(int));
+        int call2(int(*__stdcall cb)(int));
+    """)
+    lib = ffi.verify("""
+        int call1(int(*__cdecl cb)(int)) {
+            return cb(42) + 1;
+        }
+        int call2(int(*__stdcall cb)(int)) {
+            return cb(-42) - 6;
+        }
+    """)
+    xxx
+
+def test_opaque_integer_as_function_result():
+    import platform
+    if platform.machine().startswith('sparc'):
+        py.test.skip('Breaks horribly on sparc (SIGILL + corrupted stack)')
+    elif platform.machine() == 'mips64' and sys.maxsize > 2**32:
+        py.test.skip('Segfaults on mips64el')
+    # XXX bad abuse of "struct { ...; }".  It only works a bit by chance
+    # anyway.  XXX think about something better :-(
+    ffi = FFI()
+    ffi.cdef("""
+        typedef struct { ...; } myhandle_t;
+        myhandle_t foo(void);
+    """)
+    lib = ffi.verify("""
+        typedef short myhandle_t;
+        myhandle_t foo(void) { return 42; }
+    """)
+    h = lib.foo()
+    assert ffi.sizeof(h) == ffi.sizeof("short")
+
+def test_cannot_name_struct_type():
+    ffi = FFI()
+    ffi.cdef("typedef struct { int x; } *sp; void foo(sp);")
+    e = py.test.raises(VerificationError, ffi.verify,
+                       "typedef struct { int x; } *sp; void foo(sp);")
+    assert 'in argument of foo: unknown type name' in str(e.value)
+
+def test_dont_check_unnamable_fields():
+    ffi = FFI()
+    ffi.cdef("struct foo_s { struct { int x; } someone; };")
+    ffi.verify("struct foo_s { struct { int x; } someone; };")
+    # assert did not crash
+
+def test_nested_anonymous_struct_exact():
+    if sys.platform == 'win32':
+        py.test.skip("nested anonymous struct/union")
+    ffi = FFI()
+    ffi.cdef("""
+        struct foo_s { struct { int a; char b; }; union { char c, d; }; };
+    """)
+    ffi.verify("""
+        struct foo_s { struct { int a; char b; }; union { char c, d; }; };
+    """)
+    p = ffi.new("struct foo_s *")
+    assert ffi.sizeof(p[0]) == 3 * ffi.sizeof("int")    # with alignment
+    p.a = 1234567
+    p.b = b'X'
+    p.c = b'Y'
+    assert p.a == 1234567
+    assert p.b == b'X'
+    assert p.c == b'Y'
+    assert p.d == b'Y'
+
+def test_nested_anonymous_struct_exact_error():
+    if sys.platform == 'win32':
+        py.test.skip("nested anonymous struct/union")
+    ffi = FFI()
+    ffi.cdef("""
+        struct foo_s { struct { int a; char b; }; union { char c, d; }; };
+    """)
+    py.test.raises(VerificationError, ffi.verify, """
+        struct foo_s { struct { int a; short b; }; union { char c, d; }; };
+    """)
+    py.test.raises(VerificationError, ffi.verify, """
+        struct foo_s { struct { int a; char e, b; }; union { char c, d; }; };
+    """)
+
+def test_nested_anonymous_struct_inexact_1():
+    ffi = FFI()
+    ffi.cdef("""
+        struct foo_s { struct { char b; ...; }; union { char c, d; }; };
+    """)
+    ffi.verify("""
+        struct foo_s { int a, padding; char c, d, b; };
+    """)
+    assert ffi.sizeof("struct foo_s") == 3 * ffi.sizeof("int")
+
+def test_nested_anonymous_struct_inexact_2():
+    ffi = FFI()
+    ffi.cdef("""
+        struct foo_s { union { char c, d; }; struct { int a; char b; }; ...; };
+    """)
+    ffi.verify("""
+        struct foo_s { int a, padding; char c, d, b; };
+    """)
+    assert ffi.sizeof("struct foo_s") == 3 * ffi.sizeof("int")
+
+def test_ffi_union():
+    ffi = FFI()
+    ffi.cdef("union foo_u { char x; long *z; };")
+    ffi.verify("union foo_u { char x; int y; long *z; };")
+
+def test_ffi_union_partial():
+    ffi = FFI()
+    ffi.cdef("union foo_u { char x; ...; };")
+    ffi.verify("union foo_u { char x; int y; };")
+    assert ffi.sizeof("union foo_u") == 4
+
+def test_ffi_union_with_partial_struct():
+    ffi = FFI()
+    ffi.cdef("struct foo_s { int x; ...; }; union foo_u { struct foo_s s; };")
+    ffi.verify("struct foo_s { int a; int x; }; "
+               "union foo_u { char b[32]; struct foo_s s; };")
+    assert ffi.sizeof("struct foo_s") == 8
+    assert ffi.sizeof("union foo_u") == 32
+
+def test_ffi_union_partial_2():
+    ffi = FFI()
+    ffi.cdef("typedef union { char x; ...; } u1;")
+    ffi.verify("typedef union { char x; int y; } u1;")
+    assert ffi.sizeof("u1") == 4
+
+def test_ffi_union_with_partial_struct_2():
+    ffi = FFI()
+    ffi.cdef("typedef struct { int x; ...; } s1;"
+             "typedef union { s1 s; } u1;")
+    ffi.verify("typedef struct { int a; int x; } s1; "
+               "typedef union { char b[32]; s1 s; } u1;")
+    assert ffi.sizeof("s1") == 8
+    assert ffi.sizeof("u1") == 32
+    assert ffi.offsetof("u1", "s") == 0
+
+def test_ffi_struct_packed():
+    if sys.platform == 'win32':
+        py.test.skip("needs a GCC extension")
+    ffi = FFI()
+    ffi.cdef("struct foo_s { int b; ...; };")
+    ffi.verify("""
+        struct foo_s {
+            char a;
+            int b;
+        } __attribute__((packed));
+    """)
+
+def test_tmpdir():
+    import tempfile, os
+    from testing.udir import udir
+    tmpdir = tempfile.mkdtemp(dir=str(udir))
+    ffi = FFI()
+    ffi.cdef("int foo(int);")
+    lib = ffi.verify("int foo(int a) { return a + 42; }", tmpdir=tmpdir)
+    assert os.listdir(tmpdir)
+    assert lib.foo(100) == 142
+
+def test_relative_to():
+    import tempfile, os
+    from testing.udir import udir
+    tmpdir = tempfile.mkdtemp(dir=str(udir))
+    ffi = FFI()
+    ffi.cdef("int foo(int);")
+    f = open(os.path.join(tmpdir, 'foo.h'), 'w')
+    f.write("int foo(int a) { return a + 42; }\n")
+    f.close()
+    lib = ffi.verify('#include "foo.h"',
+                     include_dirs=['.'],
+                     relative_to=os.path.join(tmpdir, 'x'))
+    assert lib.foo(100) == 142
+
+def test_bug1():
+    ffi = FFI()
+    ffi.cdef("""
+        typedef struct tdlhandle_s { ...; } *tdl_handle_t;
+        typedef struct my_error_code_ {
+            tdl_handle_t *rh;
+        } my_error_code_t;
+    """)
+    ffi.verify("""
+        typedef struct tdlhandle_s { int foo; } *tdl_handle_t;
+        typedef struct my_error_code_ {
+            tdl_handle_t *rh;
+        } my_error_code_t;
+    """)
+
+def test_bool():
+    if sys.platform == 'win32':
+        py.test.skip("_Bool not in MSVC")
+    ffi = FFI()
+    ffi.cdef("struct foo_s { _Bool x; };"
+             "_Bool foo(_Bool);")
+    lib = ffi.verify("""
+        struct foo_s { _Bool x; };
+        int foo(int arg) {
+            return !arg;
+        }
+    """)
+    p = ffi.new("struct foo_s *")
+    p.x = 1
+    assert p.x == 1
+    py.test.raises(OverflowError, "p.x = -1")
+    py.test.raises(TypeError, "p.x = 0.0")
+    assert lib.foo(1) == 0
+    assert lib.foo(0) == 1
+    py.test.raises(OverflowError, lib.foo, 42)
+    py.test.raises(TypeError, lib.foo, 0.0)
+    assert int(ffi.cast("_Bool", long(1))) == 1
+    assert int(ffi.cast("_Bool", long(0))) == 0
+    assert int(ffi.cast("_Bool", long(-1))) == 1
+    assert int(ffi.cast("_Bool", 10**200)) == 1
+    assert int(ffi.cast("_Bool", 10**40000)) == 1
+    #
+    class Foo(object):
+        def __int__(self):
+            self.seen = 1
+            return result
+    f = Foo()
+    f.seen = 0
+    result = 42
+    assert int(ffi.cast("_Bool", f)) == 1
+    assert f.seen
+    f.seen = 0
+    result = 0
+    assert int(ffi.cast("_Bool", f)) == 0
+    assert f.seen
+    #
+    py.test.raises(TypeError, ffi.cast, "_Bool", [])
+
+def test_bool_on_long_double():
+    if sys.platform == 'win32':
+        py.test.skip("_Bool not in MSVC")
+    f = 1E-250
+    if f == 0.0 or f*f != 0.0:
+        py.test.skip("unexpected precision")
+    ffi = FFI()
+    ffi.cdef("long double square(long double f); _Bool opposite(_Bool);")
+    lib = ffi.verify("long double square(long double f) { return f*f; }\n"
+                     "_Bool opposite(_Bool x) { return !x; }")
+    f0 = lib.square(0.0)
+    f2 = lib.square(f)
+    f3 = lib.square(f * 2.0)
+    if repr(f2) == repr(f3):
+        py.test.skip("long double doesn't have enough precision")
+    assert float(f0) == float(f2) == float(f3) == 0.0  # too tiny for 'double'
+    assert int(ffi.cast("_Bool", f2)) == 1
+    assert int(ffi.cast("_Bool", f3)) == 1
+    assert int(ffi.cast("_Bool", f0)) == 0
+    py.test.raises(TypeError, lib.opposite, f2)
+
+def test_cannot_pass_float():
+    for basetype in ['char', 'short', 'int', 'long', 'long long']:
+        for sign in ['signed', 'unsigned']:
+            type = '%s %s' % (sign, basetype)
+            ffi = FFI()
+            ffi.cdef("struct foo_s { %s x; };\n"
+                     "int foo(%s);" % (type, type))
+            lib = ffi.verify("""
+                struct foo_s { %s x; };
+                int foo(%s arg) {
+                    return !arg;
+                }
+            """ % (type, type))
+            p = ffi.new("struct foo_s *")
+            py.test.raises(TypeError, "p.x = 0.0")
+            assert lib.foo(42) == 0
+            assert lib.foo(0) == 1
+            py.test.raises(TypeError, lib.foo, 0.0)
+
+def test_cast_from_int_type_to_bool():
+    ffi = FFI()
+    for basetype in ['char', 'short', 'int', 'long', 'long long']:
+        for sign in ['signed', 'unsigned']:
+            type = '%s %s' % (sign, basetype)
+            assert int(ffi.cast("_Bool", ffi.cast(type, 42))) == 1
+            assert int(ffi.cast("bool", ffi.cast(type, 42))) == 1
+            assert int(ffi.cast("_Bool", ffi.cast(type, 0))) == 0
+
+def test_addressof():
+    ffi = FFI()
+    ffi.cdef("""
+        struct point_s { int x, y; };
+        struct foo_s { int z; struct point_s point; };
+        struct point_s sum_coord(struct point_s *);
+    """)
+    lib = ffi.verify("""
+        struct point_s { int x, y; };
+        struct foo_s { int z; struct point_s point; };
+        struct point_s sum_coord(struct point_s *point) {
+            struct point_s r;
+            r.x = point->x + point->y;
+            r.y = point->x - point->y;
+            return r;
+        }
+    """)
+    p = ffi.new("struct foo_s *")
+    p.point.x = 16
+    p.point.y = 9
+    py.test.raises(TypeError, lib.sum_coord, p.point)
+    res = lib.sum_coord(ffi.addressof(p.point))
+    assert res.x == 25
+    assert res.y == 7
+    res2 = lib.sum_coord(ffi.addressof(res))
+    assert res2.x == 32
+    assert res2.y == 18
+    py.test.raises(TypeError, lib.sum_coord, res2)
+
+def test_callback_in_thread():
+    if sys.platform == 'win32':
+        py.test.skip("pthread only")
+    import os, subprocess, imp
+    arg = os.path.join(os.path.dirname(__file__), 'callback_in_thread.py')
+    g = subprocess.Popen([sys.executable, arg,
+                          os.path.dirname(imp.find_module('cffi')[1])])
+    result = g.wait()
+    assert result == 0
+
+def test_keepalive_lib():
+    ffi = FFI()
+    ffi.cdef("int foobar(void);")
+    lib = ffi.verify("int foobar(void) { return 42; }")
+    func = lib.foobar
+    ffi_r = weakref.ref(ffi)
+    lib_r = weakref.ref(lib)
+    del ffi
+    import gc; gc.collect()       # lib stays alive
+    assert lib_r() is not None
+    assert ffi_r() is not None
+    assert func() == 42
+
+def test_keepalive_ffi():
+    ffi = FFI()
+    ffi.cdef("int foobar(void);")
+    lib = ffi.verify("int foobar(void) { return 42; }")
+    func = lib.foobar
+    ffi_r = weakref.ref(ffi)
+    lib_r = weakref.ref(lib)
+    del lib
+    import gc; gc.collect()       # ffi stays alive
+    assert ffi_r() is not None
+    assert lib_r() is not None
+    assert func() == 42
+
+def test_FILE_stored_in_stdout():
+    if not sys.platform.startswith('linux'):
+        py.test.skip("likely, we cannot assign to stdout")
+    ffi = FFI()
+    ffi.cdef("int printf(const char *, ...); FILE *setstdout(FILE *);")
+    lib = ffi.verify("""
+        #include 
+        FILE *setstdout(FILE *f) {
+            FILE *result = stdout;
+            stdout = f;
+            return result;
+        }
+    """)
+    import os
+    fdr, fdw = os.pipe()
+    fw1 = os.fdopen(fdw, 'wb', 256)
+    old_stdout = lib.setstdout(fw1)
+    try:
+        #
+        fw1.write(b"X")
+        r = lib.printf(b"hello, %d!\n", ffi.cast("int", 42))
+        fw1.close()
+        assert r == len("hello, 42!\n")
+        #
+    finally:
+        lib.setstdout(old_stdout)
+    #
+    result = os.read(fdr, 256)
+    os.close(fdr)
+    # the 'X' might remain in the user-level buffer of 'fw1' and
+    # end up showing up after the 'hello, 42!\n'
+    assert result == b"Xhello, 42!\n" or result == b"hello, 42!\nX"
+
+def test_FILE_stored_explicitly():
+    ffi = FFI()
+    ffi.cdef("int myprintf(const char *, int); FILE *myfile;")
+    lib = ffi.verify("""
+        #include 
+        FILE *myfile;
+        int myprintf(const char *out, int value) {
+            return fprintf(myfile, out, value);
+        }
+    """)
+    import os
+    fdr, fdw = os.pipe()
+    fw1 = os.fdopen(fdw, 'wb', 256)
+    lib.myfile = ffi.cast("FILE *", fw1)
+    #
+    fw1.write(b"X")
+    r = lib.myprintf(b"hello, %d!\n", ffi.cast("int", 42))
+    fw1.close()
+    assert r == len("hello, 42!\n")
+    #
+    result = os.read(fdr, 256)
+    os.close(fdr)
+    # the 'X' might remain in the user-level buffer of 'fw1' and
+    # end up showing up after the 'hello, 42!\n'
+    assert result == b"Xhello, 42!\n" or result == b"hello, 42!\nX"
+
+def test_global_array_with_missing_length():
+    ffi = FFI()
+    ffi.cdef("int fooarray[];")
+    lib = ffi.verify("int fooarray[50];")
+    assert repr(lib.fooarray).startswith("x; }")
+    res = lib.myfunc(ffi2.new("foo_t *", {'x': 10}))
+    assert res == 420
+    res = lib.myfunc(ffi1.new("foo_t *", {'x': -10}))
+    assert res == -420
+
+def test_include_enum():
+    ffi1 = FFI()
+    ffi1.cdef("enum foo_e { AA, ... };")
+    lib1 = ffi1.verify("enum foo_e { CC, BB, AA };")
+    ffi2 = FFI()
+    ffi2.include(ffi1)
+    ffi2.cdef("int myfunc(enum foo_e);")
+    lib2 = ffi2.verify("enum foo_e { CC, BB, AA };"
+                       "int myfunc(enum foo_e x) { return (int)x; }")
+    res = lib2.myfunc(lib2.AA)
+    assert res == 2
+
+def test_enum_size():
+    cases = [('123',           4, 4294967295),
+             ('4294967295U',   4, 4294967295),
+             ('-123',          4, -1),
+             ('-2147483647-1', 4, -1),
+             ]
+    if FFI().sizeof("long") == 8:
+        cases += [('4294967296L',        8, 2**64-1),
+                  ('%dUL' % (2**64-1),   8, 2**64-1),
+                  ('-2147483649L',       8, -1),
+                  ('%dL-1L' % (1-2**63), 8, -1)]
+    for hidden_value, expected_size, expected_minus1 in cases:
+        if sys.platform == 'win32' and 'U' in hidden_value:
+            continue   # skipped on Windows
+        ffi = FFI()
+        ffi.cdef("enum foo_e { AA, BB, ... };")
+        lib = ffi.verify("enum foo_e { AA, BB=%s };" % hidden_value)
+        assert lib.AA == 0
+        assert lib.BB == eval(hidden_value.replace('U', '').replace('L', ''))
+        assert ffi.sizeof("enum foo_e") == expected_size
+        assert int(ffi.cast("enum foo_e", -1)) == expected_minus1
+    # test with the large value hidden:
+    # disabled so far, doesn't work
+##    for hidden_value, expected_size, expected_minus1 in cases:
+##        ffi = FFI()
+##        ffi.cdef("enum foo_e { AA, BB, ... };")
+##        lib = ffi.verify("enum foo_e { AA, BB=%s };" % hidden_value)
+##        assert lib.AA == 0
+##        assert ffi.sizeof("enum foo_e") == expected_size
+##        assert int(ffi.cast("enum foo_e", -1)) == expected_minus1
+
+def test_enum_bug118():
+    maxulong = 256 ** FFI().sizeof("unsigned long") - 1
+    for c1, c2, c2c in [(0xffffffff, -1, ''),
+                        (maxulong, -1, ''),
+                        (-1, 0xffffffff, 'U'),
+                        (-1, maxulong, 'UL')]:
+        if c2c and sys.platform == 'win32':
+            continue     # enums may always be signed with MSVC
+        ffi = FFI()
+        ffi.cdef("enum foo_e { AA=%s };" % c1)
+        e = py.test.raises(VerificationError, ffi.verify,
+                           "enum foo_e { AA=%s%s };" % (c2, c2c))
+        assert str(e.value) == ('enum foo_e: AA has the real value %d, not %d'
+                                % (c2, c1))
+
+def test_string_to_voidp_arg():
+    ffi = FFI()
+    ffi.cdef("int myfunc(void *);")
+    lib = ffi.verify("int myfunc(void *p) { return ((signed char *)p)[0]; }")
+    res = lib.myfunc(b"hi!")
+    assert res == ord(b"h")
+    p = ffi.new("char[]", b"gah")
+    res = lib.myfunc(p)
+    assert res == ord(b"g")
+    res = lib.myfunc(ffi.cast("void *", p))
+    assert res == ord(b"g")
+    res = lib.myfunc(ffi.cast("int *", p))
+    assert res == ord(b"g")
+
+def test_callback_indirection():
+    ffi = FFI()
+    ffi.cdef("""
+        int (*python_callback)(int how_many, int *values);
+        int (*const c_callback)(int,...);   /* pass this ptr to C routines */
+        int some_c_function(int(*cb)(int,...));
+    """)
+    lib = ffi.verify("""
+        #include 
+        #ifdef _WIN32
+        #include 
+        #define alloca _alloca
+        #else
+        # ifdef __FreeBSD__
+        #  include 
+        # else
+        #  include 
+        # endif
+        #endif
+        static int (*python_callback)(int how_many, int *values);
+        static int c_callback(int how_many, ...) {
+            va_list ap;
+            /* collect the "..." arguments into the values[] array */
+            int i, *values = alloca((size_t)how_many * sizeof(int));
+            va_start(ap, how_many);
+            for (i=0; i"
+
+def test_bug_const_char_ptr_array_1():
+    ffi = FFI()
+    ffi.cdef("""const char *a[...];""")
+    lib = ffi.verify("""const char *a[5];""")
+    assert repr(ffi.typeof(lib.a)) == ""
+
+def test_bug_const_char_ptr_array_2():
+    from cffi import FFI     # ignore warnings
+    ffi = FFI()
+    ffi.cdef("""const int a[];""")
+    lib = ffi.verify("""const int a[5];""")
+    assert repr(ffi.typeof(lib.a)) == ""
+
+def _test_various_calls(force_libffi):
+    cdef_source = """
+    int xvalue;
+    long long ivalue, rvalue;
+    float fvalue;
+    double dvalue;
+    long double Dvalue;
+    signed char tf_bb(signed char x, signed char c);
+    unsigned char tf_bB(signed char x, unsigned char c);
+    short tf_bh(signed char x, short c);
+    unsigned short tf_bH(signed char x, unsigned short c);
+    int tf_bi(signed char x, int c);
+    unsigned int tf_bI(signed char x, unsigned int c);
+    long tf_bl(signed char x, long c);
+    unsigned long tf_bL(signed char x, unsigned long c);
+    long long tf_bq(signed char x, long long c);
+    unsigned long long tf_bQ(signed char x, unsigned long long c);
+    float tf_bf(signed char x, float c);
+    double tf_bd(signed char x, double c);
+    long double tf_bD(signed char x, long double c);
+    """
+    if force_libffi:
+        cdef_source = (cdef_source
+            .replace('tf_', '(*const tf_')
+            .replace('(signed char x', ')(signed char x'))
+    ffi = FFI()
+    ffi.cdef(cdef_source)
+    lib = ffi.verify("""
+    int xvalue;
+    long long ivalue, rvalue;
+    float fvalue;
+    double dvalue;
+    long double Dvalue;
+
+    typedef signed char b_t;
+    typedef unsigned char B_t;
+    typedef short h_t;
+    typedef unsigned short H_t;
+    typedef int i_t;
+    typedef unsigned int I_t;
+    typedef long l_t;
+    typedef unsigned long L_t;
+    typedef long long q_t;
+    typedef unsigned long long Q_t;
+    typedef float f_t;
+    typedef double d_t;
+    typedef long double D_t;
+    #define S(letter)  xvalue = (int)x; letter##value = (letter##_t)c;
+    #define R(letter)  return (letter##_t)rvalue;
+
+    signed char tf_bb(signed char x, signed char c) { S(i) R(b) }
+    unsigned char tf_bB(signed char x, unsigned char c) { S(i) R(B) }
+    short tf_bh(signed char x, short c) { S(i) R(h) }
+    unsigned short tf_bH(signed char x, unsigned short c) { S(i) R(H) }
+    int tf_bi(signed char x, int c) { S(i) R(i) }
+    unsigned int tf_bI(signed char x, unsigned int c) { S(i) R(I) }
+    long tf_bl(signed char x, long c) { S(i) R(l) }
+    unsigned long tf_bL(signed char x, unsigned long c) { S(i) R(L) }
+    long long tf_bq(signed char x, long long c) { S(i) R(q) }
+    unsigned long long tf_bQ(signed char x, unsigned long long c) { S(i) R(Q) }
+    float tf_bf(signed char x, float c) { S(f) R(f) }
+    double tf_bd(signed char x, double c) { S(d) R(d) }
+    long double tf_bD(signed char x, long double c) { S(D) R(D) }
+    """)
+    lib.rvalue = 0x7182838485868788
+    for kind, cname in [('b', 'signed char'),
+                        ('B', 'unsigned char'),
+                        ('h', 'short'),
+                        ('H', 'unsigned short'),
+                        ('i', 'int'),
+                        ('I', 'unsigned int'),
+                        ('l', 'long'),
+                        ('L', 'unsigned long'),
+                        ('q', 'long long'),
+                        ('Q', 'unsigned long long'),
+                        ('f', 'float'),
+                        ('d', 'double'),
+                        ('D', 'long double')]:
+        sign = +1 if 'unsigned' in cname else -1
+        lib.xvalue = 0
+        lib.ivalue = 0
+        lib.fvalue = 0
+        lib.dvalue = 0
+        lib.Dvalue = 0
+        fun = getattr(lib, 'tf_b' + kind)
+        res = fun(-42, sign * 99)
+        if kind == 'D':
+            res = float(res)
+        assert res == int(ffi.cast(cname, 0x7182838485868788))
+        assert lib.xvalue == -42
+        if kind in 'fdD':
+            assert float(getattr(lib, kind + 'value')) == -99.0
+        else:
+            assert lib.ivalue == sign * 99
+
+def test_various_calls_direct():
+    _test_various_calls(force_libffi=False)
+
+def test_various_calls_libffi():
+    _test_various_calls(force_libffi=True)
+
+def test_ptr_to_opaque():
+    ffi = FFI()
+    ffi.cdef("typedef ... foo_t; int f1(foo_t*); foo_t *f2(int);")
+    lib = ffi.verify("""
+        #include 
+        typedef struct { int x; } foo_t;
+        int f1(foo_t* p) {
+            int x = p->x;
+            free(p);
+            return x;
+        }
+        foo_t *f2(int x) {
+            foo_t *p = malloc(sizeof(foo_t));
+            p->x = x;
+            return p;
+        }
+    """)
+    p = lib.f2(42)
+    x = lib.f1(p)
+    assert x == 42
+
+def _run_in_multiple_threads(test1):
+    test1()
+    import sys
+    try:
+        import thread
+    except ImportError:
+        import _thread as thread
+    errors = []
+    def wrapper(lock):
+        try:
+            test1()
+        except:
+            errors.append(sys.exc_info())
+        lock.release()
+    locks = []
+    for i in range(10):
+        _lock = thread.allocate_lock()
+        _lock.acquire()
+        thread.start_new_thread(wrapper, (_lock,))
+        locks.append(_lock)
+    for _lock in locks:
+        _lock.acquire()
+        if errors:
+            raise errors[0][1]
+
+def test_errno_working_even_with_pypys_jit():
+    ffi = FFI()
+    ffi.cdef("int f(int);")
+    lib = ffi.verify("""
+        #include 
+        int f(int x) { return (errno = errno + x); }
+    """)

From noreply at buildbot.pypy.org  Sat Apr 18 17:33:23 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 18 Apr 2015 17:33:23 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Small fixes to have a few tests pass.
Message-ID: <20150418153323.10A0C1C0484@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1759:e3236c67b86f
Date: 2015-04-18 17:27 +0200
http://bitbucket.org/cffi/cffi/changeset/e3236c67b86f/

Log:	Small fixes to have a few tests pass.

diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -321,7 +321,9 @@
 
     def verify(self, source='', **kwargs):
         from recompiler import verify    # XXX must be in the current dir
-        return verify(self, 'verify', source, **kwargs)
+        FFI._verify_counter += 1
+        return verify(self, 'verify%d' % FFI._verify_counter, source, **kwargs)
+    _verify_counter = 0
 
     def XXXverify(self, source='', tmpdir=None, **kwargs):
         """Verify that the current ffi signatures compile on this
diff --git a/new/recompiler.py b/new/recompiler.py
--- a/new/recompiler.py
+++ b/new/recompiler.py
@@ -534,6 +534,9 @@
     # ----------
     # emitting the opcodes for individual types
 
+    def _emit_bytecode_VoidType(self, tp, index):
+        self.cffi_types[index] = CffiOp(OP_PRIMITIVE, PRIM_VOID)
+
     def _emit_bytecode_PrimitiveType(self, tp, index):
         prim_index = PRIMITIVE_TO_INDEX[tp.name]
         self.cffi_types[index] = CffiOp(OP_PRIMITIVE, prim_index)
diff --git a/new/test_verify1.py b/new/test_verify1.py
--- a/new/test_verify1.py
+++ b/new/test_verify1.py
@@ -1,6 +1,8 @@
-import sys
+import sys, math, py
 from cffi1 import FFI, VerificationError, model
 
+lib_m = ['m']
+
 
 def test_missing_function(ffi=None):
     # uses the FFI hacked above with '-Werror'
@@ -9,7 +11,7 @@
     ffi.cdef("void some_completely_unknown_function();")
     try:
         lib = ffi.verify()
-    except (VerificationError, OSError):
+    except (VerificationError, OSError, ImportError):
         pass     # expected case: we get a VerificationError
     else:
         # but depending on compiler and loader details, maybe
@@ -20,8 +22,8 @@
 
 def test_missing_function_import_error():
     # uses the original FFI that just gives a warning during compilation
-    import cffi
-    test_missing_function(ffi=cffi.FFI())
+    import cffi1
+    test_missing_function(ffi=cffi1.FFI())
 
 def test_simple_case():
     ffi = FFI()

From noreply at buildbot.pypy.org  Sat Apr 18 17:33:24 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 18 Apr 2015 17:33:24 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: keep this from test_verify.py
Message-ID: <20150418153324.2B5171C0484@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1760:0528f5ac101b
Date: 2015-04-18 17:29 +0200
http://bitbucket.org/cffi/cffi/changeset/0528f5ac101b/

Log:	keep this from test_verify.py

diff --git a/new/test_verify1.py b/new/test_verify1.py
--- a/new/test_verify1.py
+++ b/new/test_verify1.py
@@ -2,6 +2,27 @@
 from cffi1 import FFI, VerificationError, model
 
 lib_m = ['m']
+if sys.platform == 'win32':
+    #there is a small chance this fails on Mingw via environ $CC
+    import distutils.ccompiler
+    if distutils.ccompiler.get_default_compiler() == 'msvc':
+        lib_m = ['msvcrt']
+    pass      # no obvious -Werror equivalent on MSVC
+else:
+    if (sys.platform == 'darwin' and
+          [int(x) for x in os.uname()[2].split('.')] >= [11, 0, 0]):
+        # assume a standard clang or gcc
+        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion']
+        # special things for clang
+        extra_compile_args.append('-Qunused-arguments')
+    else:
+        # assume a standard gcc
+        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion']
+
+    class FFI(FFI):
+        def verify(self, *args, **kwds):
+            return super(FFI, self).verify(
+                *args, extra_compile_args=extra_compile_args, **kwds)
 
 
 def test_missing_function(ffi=None):

From noreply at buildbot.pypy.org  Sat Apr 18 17:33:25 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 18 Apr 2015 17:33:25 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Disable most methods from the pure
	Python class FFI
Message-ID: <20150418153325.317B71C0484@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1761:1c56d670c261
Date: 2015-04-18 17:31 +0200
http://bitbucket.org/cffi/cffi/changeset/1c56d670c261/

Log:	Disable most methods from the pure Python class FFI

diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -142,7 +142,7 @@
                             "pointer-to-function type" % (cdecl,))
         return btype
 
-    def typeof(self, cdecl):
+    def XXXtypeof(self, cdecl):
         """Parse the C type given as a string and return the
         corresponding  object.
         It can also be used on 'cdata' instance to get its C type.
@@ -161,7 +161,7 @@
                 return self._get_cached_btype(cdecl._cffi_base_type)
         raise TypeError(type(cdecl))
 
-    def sizeof(self, cdecl):
+    def XXXsizeof(self, cdecl):
         """Return the size in bytes of the argument.  It can be a
         string naming a C type, or a 'cdata' instance.
         """
@@ -171,7 +171,7 @@
         else:
             return self._backend.sizeof(cdecl)
 
-    def alignof(self, cdecl):
+    def XXXalignof(self, cdecl):
         """Return the natural alignment size in bytes of the C type
         given as a string.
         """
@@ -179,7 +179,7 @@
             cdecl = self._typeof(cdecl)
         return self._backend.alignof(cdecl)
 
-    def offsetof(self, cdecl, *fields_or_indexes):
+    def XXXoffsetof(self, cdecl, *fields_or_indexes):
         """Return the offset of the named field inside the given
         structure or array, which must be given as a C type name.  
         You can give several field names in case of nested structures.
@@ -217,7 +217,7 @@
             cdecl = self._typeof(cdecl)
         return self._backend.newp(cdecl, init)
 
-    def cast(self, cdecl, source):
+    def XXXcast(self, cdecl, source):
         """Similar to a C cast: returns an instance of the named C
         type initialized with the given 'source'.  The source is
         casted between integers or pointers of any type.
@@ -244,7 +244,7 @@
         """
         return self._backend.string(cdata, maxlen)
 
-    def buffer(self, cdata, size=-1):
+    def XXXbuffer(self, cdata, size=-1):
         """Return a read-write buffer object that references the raw C data
         pointed to by the given 'cdata'.  The 'cdata' must be a pointer or
         an array.  Can be passed to functions expecting a buffer, or directly
@@ -257,7 +257,7 @@
         """
         return self._backend.buffer(cdata, size)
 
-    def from_buffer(self, python_buffer):
+    def XXXfrom_buffer(self, python_buffer):
         """Return a  that points to the data of the
         given Python object, which must support the buffer interface.
         Note that this is not meant to be used on the built-in types str,
@@ -267,7 +267,7 @@
         """
         return self._backend.from_buffer(self.BCharA, python_buffer)
 
-    def callback(self, cdecl, python_callable=None, error=None):
+    def XXXcallback(self, cdecl, python_callable=None, error=None):
         """Return a callback object or a decorator making such a
         callback object.  'cdecl' must name a C function pointer type.
         The callback invokes the specified 'python_callable' (which may
@@ -287,7 +287,7 @@
         else:
             return callback_decorator_wrap(python_callable)  # direct mode
 
-    def getctype(self, cdecl, replace_with=''):
+    def XXXgetctype(self, cdecl, replace_with=''):
         """Return a string giving the C type 'cdecl', which may be itself
         a string or a  object.  If 'replace_with' is given, it gives
         extra text to append (or insert for more complicated C types), like
@@ -303,7 +303,7 @@
             replace_with = ' ' + replace_with
         return self._backend.getcname(cdecl, replace_with)
 
-    def gc(self, cdata, destructor):
+    def XXXgc(self, cdata, destructor):
         """Return a new cdata object that points to the same
         data.  Later, when this new cdata object is garbage-collected,
         'destructor(old_cdata_object)' will be called.
@@ -355,39 +355,14 @@
         self._libraries.append(lib)
         return lib
 
-    def csource(self, modulename, source='', **kwargs):
-        if hasattr(self, '_csource'):
-            raise ValueError("can't call csource() more than once")
-        self._csource = (modulename, source, kwargs)
-
-    def recompile(self):
-        from .verifier import Verifier
-        modulename, source, kwargs = self._csource
-        if self._windows_unicode:
-            self._apply_windows_unicode(kwargs)
-        verifier = Verifier(self, source, modulename=modulename+'_ffi',
-                            tmpdir='.',  **kwargs)
-        verifier.compile_module()
-        f = open(modulename + '.py', 'w')
-        print >> f, 'import os, cffi as _cffi'
-        print >> f, 'ffi = _cffi.FFI()'
-        for cdef in self._cdefsources:
-            print >> f
-            print >> f, 'ffi.cdef(%r)' % (cdef,)
-        print >> f
-        print >> f, 'lib = ffi.verify('
-        print >> f, '    tmpdir=os.path.dirname(__file__), modulename=%r)' % (
-            modulename + '_ffi',)
-        f.close()
-
     def _get_errno(self):
         return self._backend.get_errno()
     def _set_errno(self, errno):
         self._backend.set_errno(errno)
-    errno = property(_get_errno, _set_errno, None,
+    XXXerrno = property(_get_errno, _set_errno, None,
                      "the value of 'errno' from/to the C calls")
 
-    def getwinerror(self, code=-1):
+    def XXXgetwinerror(self, code=-1):
         return self._backend.getwinerror(code)
 
     def _pointer_to(self, ctype):
@@ -395,7 +370,7 @@
         with self._lock:
             return model.pointer_cache(self, ctype)
 
-    def addressof(self, cdata, *fields_or_indexes):
+    def XXXaddressof(self, cdata, *fields_or_indexes):
         """Return the address of a .
         If 'fields_or_indexes' are given, returns the address of that
         field or array item in the structure or array, recursively in
@@ -427,6 +402,7 @@
         variables, which must anyway be accessed directly from the
         lib object returned by the original FFI instance.
         """
+        XXX
         with ffi_to_include._lock:
             with self._lock:
                 self._parser.include(ffi_to_include._parser)
@@ -434,10 +410,10 @@
                 self._cdefsources.extend(ffi_to_include._cdefsources)
                 self._cdefsources.append(']')
 
-    def new_handle(self, x):
+    def XXXnew_handle(self, x):
         return self._backend.newp_handle(self.BVoidP, x)
 
-    def from_handle(self, x):
+    def XXXfrom_handle(self, x):
         return self._backend.from_handle(x)
 
     def set_unicode(self, enabled_flag):
@@ -447,6 +423,7 @@
         declare these types to be (pointers to) plain 8-bit characters.
         This is mostly for backward compatibility; you usually want True.
         """
+        XXX
         if self._windows_unicode is not None:
             raise ValueError("set_unicode() can only be called once")
         enabled_flag = bool(enabled_flag)

From noreply at buildbot.pypy.org  Sat Apr 18 17:33:26 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 18 Apr 2015 17:33:26 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: ffi.cast()
Message-ID: <20150418153326.33AAF1C0484@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1762:3fbd5a5e1309
Date: 2015-04-18 17:33 +0200
http://bitbucket.org/cffi/cffi/changeset/3fbd5a5e1309/

Log:	ffi.cast()

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -3062,13 +3062,9 @@
     return cd;
 }
 
-static PyObject *b_cast(PyObject *self, PyObject *args)
-{
-    CTypeDescrObject *ct;
+static PyObject *do_cast(CTypeDescrObject *ct, PyObject *ob)
+{
     CDataObject *cd;
-    PyObject *ob;
-    if (!PyArg_ParseTuple(args, "O!O:cast", &CTypeDescr_Type, &ct, &ob))
-        return NULL;
 
     if (ct->ct_flags & (CT_POINTER|CT_FUNCTIONPTR|CT_ARRAY) &&
         ct->ct_size >= 0) {
@@ -3185,6 +3181,16 @@
     return NULL;
 }
 
+static PyObject *b_cast(PyObject *self, PyObject *args)
+{
+    CTypeDescrObject *ct;
+    PyObject *ob;
+    if (!PyArg_ParseTuple(args, "O!O:cast", &CTypeDescr_Type, &ct, &ob))
+        return NULL;
+
+    return do_cast(ct, ob);
+}
+
 /************************************************************/
 
 typedef struct {
diff --git a/new/ffi_obj.c b/new/ffi_obj.c
--- a/new/ffi_obj.c
+++ b/new/ffi_obj.c
@@ -244,8 +244,7 @@
     return direct_newp(ct, init);
 }
 
-#if 0
-static PyObject *ffi_cast(ZefFFIObject *self, PyObject *args)
+static PyObject *ffi_cast(FFIObject *self, PyObject *args)
 {
     CTypeDescrObject *ct;
     PyObject *ob, *arg;
@@ -258,7 +257,6 @@
 
     return do_cast(ct, ob);
 }
-#endif
 
 PyDoc_STRVAR(ffi_string_doc,
 "Return a Python string (or unicode string) from the 'cdata'.  If\n"
@@ -522,7 +520,9 @@
 static PyMethodDef ffi_methods[] = {
 #if 0
     {"addressof",     (PyCFunction)ffi_addressof, METH_VARARGS},
+#endif
     {"cast",          (PyCFunction)ffi_cast,      METH_VARARGS},
+#if 0
     {"close_library", ffi_close_library,          METH_VARARGS | METH_STATIC},
     {"from_handle",   (PyCFunction)ffi_from_handle,METH_O},
     {"gc",            (PyCFunction)ffi_gc,        METH_VARARGS},

From noreply at buildbot.pypy.org  Sat Apr 18 17:46:32 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 18 Apr 2015 17:46:32 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: ffi.cast() and fix (can't use any
 other bits of ml_flags)
Message-ID: <20150418154632.B0AC01C0484@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1763:ef894ff04dea
Date: 2015-04-18 17:47 +0200
http://bitbucket.org/cffi/cffi/changeset/ef894ff04dea/

Log:	ffi.cast() and fix (can't use any other bits of ml_flags)

diff --git a/new/ffi_obj.c b/new/ffi_obj.c
--- a/new/ffi_obj.c
+++ b/new/ffi_obj.c
@@ -244,6 +244,11 @@
     return direct_newp(ct, init);
 }
 
+PyDoc_STRVAR(ffi_cast_doc,
+"Similar to a C cast: returns an instance of the named C\n"
+"type initialized with the given 'source'.  The source is\n"
+"casted between integers or pointers of any type.");
+
 static PyObject *ffi_cast(FFIObject *self, PyObject *args)
 {
     CTypeDescrObject *ct;
@@ -521,7 +526,7 @@
 #if 0
     {"addressof",     (PyCFunction)ffi_addressof, METH_VARARGS},
 #endif
-    {"cast",          (PyCFunction)ffi_cast,      METH_VARARGS},
+    {"cast",          (PyCFunction)ffi_cast,      METH_VARARGS, ffi_cast_doc},
 #if 0
     {"close_library", ffi_close_library,          METH_VARARGS | METH_STATIC},
     {"from_handle",   (PyCFunction)ffi_from_handle,METH_O},
diff --git a/new/lib_obj.c b/new/lib_obj.c
--- a/new/lib_obj.c
+++ b/new/lib_obj.c
@@ -15,8 +15,8 @@
     PyMethodDef md;
     int type_index;
 };
-
-#define METH_CPYEXTFUNC   0x40000000
+static const char cpyextfunc_doc[] =
+    "direct call to the C function of the same name";
 
 struct LibObject_s {
     PyObject_HEAD
@@ -29,22 +29,23 @@
 
 static PyObject *_cpyextfunc_type_index(PyObject *x)
 {
+    struct CPyExtFunc_s *exf;
     assert(PyErr_Occurred());
 
     if (!PyCFunction_Check(x))
         return NULL;
     if (!LibObject_Check(PyCFunction_GET_SELF(x)))
         return NULL;
-    if (!(PyCFunction_GET_FLAGS(x) & METH_CPYEXTFUNC))
+
+    exf = (struct CPyExtFunc_s *)(((PyCFunctionObject *)x) -> m_ml);
+    if (exf->md.ml_doc != cpyextfunc_doc)
         return NULL;
 
     PyErr_Clear();
 
     LibObject *lib = (LibObject *)PyCFunction_GET_SELF(x);
-    struct CPyExtFunc_s *exf;
     PyObject *tuple, *result;
 
-    exf = (struct CPyExtFunc_s *)(((PyCFunctionObject *)x) -> m_ml);
     tuple = _realize_c_type_or_func(lib->l_types_builder,
                                     lib->l_types_builder->ctx.types,
                                     exf->type_index);
@@ -112,9 +113,9 @@
         goto no_memory;
 
     xfunc->md.ml_meth = (PyCFunction)g->address;
-    xfunc->md.ml_flags = flags | METH_CPYEXTFUNC;
+    xfunc->md.ml_flags = flags;
     xfunc->md.ml_name = g->name;
-    /*xfunc->md.ml_doc = ... */
+    xfunc->md.ml_doc = cpyextfunc_doc;
     if (xfunc->md.ml_name == NULL)
         goto no_memory;
 

From noreply at buildbot.pypy.org  Sat Apr 18 17:58:39 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 18 Apr 2015 17:58:39 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: fix
Message-ID: <20150418155839.8DC041C0484@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1764:82a3a96c61a1
Date: 2015-04-18 17:51 +0200
http://bitbucket.org/cffi/cffi/changeset/82a3a96c61a1/

Log:	fix

diff --git a/new/test_verify1.py b/new/test_verify1.py
--- a/new/test_verify1.py
+++ b/new/test_verify1.py
@@ -24,6 +24,12 @@
             return super(FFI, self).verify(
                 *args, extra_compile_args=extra_compile_args, **kwds)
 
+class U(object):
+    def __add__(self, other):
+        return eval('u'+repr(other).replace(r'\\u', r'\u')
+                                   .replace(r'\\U', r'\U'))
+u = U()
+
 
 def test_missing_function(ffi=None):
     # uses the FFI hacked above with '-Werror'

From noreply at buildbot.pypy.org  Sat Apr 18 17:58:40 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 18 Apr 2015 17:58:40 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: fix for compiler warnings
Message-ID: <20150418155840.93F021C0484@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1765:8a35ce16a595
Date: 2015-04-18 17:51 +0200
http://bitbucket.org/cffi/cffi/changeset/8a35ce16a595/

Log:	fix for compiler warnings

diff --git a/new/recompiler.py b/new/recompiler.py
--- a/new/recompiler.py
+++ b/new/recompiler.py
@@ -433,7 +433,7 @@
                 '    offsetof(struct _cffi_align_%s, y),\n' % (name,) +
                 '    _cffi_FIELDS_FOR_%s, %d },' % (name, len(tp.fldtypes),))
         else:
-            size_align = ' -1, -1, -1, 0 /* opaque */ },'
+            size_align = ' (size_t)-1, -1, -1, 0 /* opaque */ },'
         self._lsts["struct_union"].append(
             '  { "%s", %d, %s,' % (name, type_index, flags) + size_align)
 

From noreply at buildbot.pypy.org  Sat Apr 18 17:58:41 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 18 Apr 2015 17:58:41 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Copy from vengine_cpy
Message-ID: <20150418155841.8D92B1C0484@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1766:947a567c3565
Date: 2015-04-18 17:59 +0200
http://bitbucket.org/cffi/cffi/changeset/947a567c3565/

Log:	Copy from vengine_cpy

diff --git a/new/recompiler.py b/new/recompiler.py
--- a/new/recompiler.py
+++ b/new/recompiler.py
@@ -400,11 +400,31 @@
                 self._do_collect_type(self._field_type(tp, name1, tp1))
 
     def _generate_cpy_struct_decl(self, tp, name):
-        if tp.fldtypes is not None:
-            prnt = self._prnt
-            prnt("struct _cffi_align_%s { char x; %s %s y; };" % (
-                name, tp.kind, name))
-            prnt()
+        if tp.fldtypes is None:
+            return
+        cname = ('%s %s' % (tp.kind, name)).strip()
+        prnt = self._prnt
+        checkfuncname = '_cffi_checkfld_%s' % (name,)
+        prnt('__attribute__((unused))')
+        prnt('static void %s(%s *p)' % (checkfuncname, cname))
+        prnt('{')
+        prnt('  /* only to generate compile-time warnings or errors */')
+        prnt('  (void)p;')
+        for fname, ftype, fbitsize in tp.enumfields():
+            if (isinstance(ftype, model.PrimitiveType)
+                and ftype.is_integer_type()) or fbitsize >= 0:
+                # accept all integers, but complain on float or double
+                prnt('  (void)((p->%s) << 1);' % fname)
+            else:
+                # only accept exactly the type declared.
+                try:
+                    prnt('  { %s = &p->%s; (void)tmp; }' % (
+                        ftype.get_c_name('*tmp', 'field %r'%fname), fname))
+                except ffiplatform.VerificationError as e:
+                    prnt('  /* %s */' % str(e))   # cannot verify it, ignore
+        prnt('}')
+        prnt('struct _cffi_align_%s { char x; %s y; };' % (name, cname))
+        prnt()
 
     def _generate_cpy_struct_ctx(self, tp, name):
         type_index = self._typesdict[tp]

From noreply at buildbot.pypy.org  Sat Apr 18 18:57:35 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Sat, 18 Apr 2015 18:57:35 +0200 (CEST)
Subject: [pypy-commit] pypy callfamily: cleanup
Message-ID: <20150418165735.B4D011C0484@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: callfamily
Changeset: r76825:90065281d228
Date: 2015-04-15 20:39 +0100
http://bitbucket.org/pypy/pypy/changeset/90065281d228/

Log:	cleanup

diff --git a/rpython/rtyper/rpbc.py b/rpython/rtyper/rpbc.py
--- a/rpython/rtyper/rpbc.py
+++ b/rpython/rtyper/rpbc.py
@@ -1,6 +1,8 @@
 import types
 
-from rpython.annotator import model as annmodel, description
+from rpython.annotator import model as annmodel
+from rpython.annotator.description import (
+    FunctionDesc, ClassDesc, MethodDesc, FrozenDesc, MethodOfFrozenDesc)
 from rpython.flowspace.model import Constant
 from rpython.annotator.argument import simple_args
 from rpython.rtyper import rclass, callparse
@@ -16,7 +18,8 @@
     if 1 < len(s_pbc.descriptions) < rtyper.getconfig().translation.withsmallfuncsets:
         callfamily = s_pbc.any_description().getcallfamily()
         concretetable, uniquerows = get_concrete_calltable(rtyper, callfamily)
-        if len(uniquerows) == 1 and (not s_pbc.subset_of or small_cand(rtyper, s_pbc.subset_of)):
+        if (len(uniquerows) == 1 and
+                (not s_pbc.subset_of or small_cand(rtyper, s_pbc.subset_of))):
             return True
     return False
 
@@ -25,7 +28,7 @@
         from rpython.rtyper.lltypesystem.rpbc import (
             FunctionsPBCRepr, SmallFunctionSetPBCRepr)
         kind = self.getKind()
-        if issubclass(kind, description.FunctionDesc):
+        if issubclass(kind, FunctionDesc):
             sample = self.any_description()
             callfamily = sample.querycallfamily()
             if callfamily and callfamily.total_calltable_size > 0:
@@ -34,14 +37,14 @@
                     getRepr = SmallFunctionSetPBCRepr
             else:
                 getRepr = getFrozenPBCRepr
-        elif issubclass(kind, description.ClassDesc):
+        elif issubclass(kind, ClassDesc):
             # user classes
             getRepr = ClassesPBCRepr
-        elif issubclass(kind, description.MethodDesc):
+        elif issubclass(kind, MethodDesc):
             getRepr = MethodsPBCRepr
-        elif issubclass(kind, description.FrozenDesc):
+        elif issubclass(kind, FrozenDesc):
             getRepr = getFrozenPBCRepr
-        elif issubclass(kind, description.MethodOfFrozenDesc):
+        elif issubclass(kind, MethodOfFrozenDesc):
             getRepr = MethodOfFrozenPBCRepr
         else:
             raise TyperError("unexpected PBC kind %r" % (kind,))
@@ -55,7 +58,7 @@
             t = self.subset_of.rtyper_makekey()
         else:
             t = ()
-        return tuple([self.__class__, self.can_be_None]+lst)+t
+        return tuple([self.__class__, self.can_be_None] + lst) + t
 
 # ____________________________________________________________
 
@@ -111,8 +114,8 @@
                 llfn = rtyper.getcallable(graph)
                 concreterow[funcdesc] = llfn
             assert len(concreterow) > 0
-            concreterow.fntype = lltype.typeOf(llfn)# 'llfn' from the loop above
-                                         # (they should all have the same type)
+            # 'llfn' should be the same for all graphs
+            concreterow.fntype = lltype.typeOf(llfn)
             concreterows[shape, index] = concreterow
 
     for row in concreterows.values():
@@ -224,9 +227,9 @@
 
     def convert_const(self, value):
         if isinstance(value, types.MethodType) and value.im_self is None:
-            value = value.im_func   # unbound method -> bare function
+            value = value.im_func  # unbound method -> bare function
         elif isinstance(value, staticmethod):
-            value = value.__get__(42) # hackish, get the function wrapped by staticmethod
+            value = value.__get__(42)  # hackish, get the function wrapped by staticmethod
         if self.lowleveltype is lltype.Void:
             return None
         if value is None:
@@ -280,7 +283,7 @@
         if not graphs:
             raise TyperError("cannot pass here a function that is not called")
         graph = graphs[0]
-        if graphs != [graph]*len(graphs):
+        if graphs != [graph] * len(graphs):
             raise TyperError("cannot pass a specialized function here")
         llfn = self.rtyper.getcallable(graph)
         return inputconst(lltype.typeOf(llfn), llfn)
@@ -288,7 +291,7 @@
     def get_concrete_llfn(self, s_pbc, args_s, op):
         bk = self.rtyper.annotator.bookkeeper
         descs = list(s_pbc.descriptions)
-        vfcs = description.FunctionDesc.variant_for_call_site
+        vfcs = FunctionDesc.variant_for_call_site
         args = simple_args(args_s)
         shape, index = vfcs(bk, self.callfamily, descs, args, op)
         funcdesc, = descs
@@ -308,7 +311,7 @@
         args = hop.spaceop.build_args(hop.args_s[1:])
         s_pbc = hop.args_s[0]   # possibly more precise than self.s_pbc
         descs = list(s_pbc.descriptions)
-        vfcs = description.FunctionDesc.variant_for_call_site
+        vfcs = FunctionDesc.variant_for_call_site
         shape, index = vfcs(bk, self.callfamily, descs, args, hop.spaceop)
         row_of_graphs = self.callfamily.calltables[shape][index]
         anygraph = row_of_graphs.itervalues().next()  # pick any witness
@@ -319,10 +322,10 @@
         rresult = callparse.getrresult(self.rtyper, anygraph)
         hop.exception_is_here()
         if isinstance(vlist[0], Constant):
-            v = hop.genop('direct_call', vlist, resulttype = rresult)
+            v = hop.genop('direct_call', vlist, resulttype=rresult)
         else:
             vlist.append(hop.inputconst(lltype.Void, row_of_graphs.values()))
-            v = hop.genop('indirect_call', vlist, resulttype = rresult)
+            v = hop.genop('indirect_call', vlist, resulttype=rresult)
         if hop.r_result is impossible_repr:
             return None      # see test_always_raising_methods
         else:
@@ -419,8 +422,6 @@
     def convert_const(self, pbc):
         if pbc is None:
             return self.null_instance()
-        if isinstance(pbc, types.MethodType) and pbc.im_self is None:
-            value = pbc.im_func   # unbound method -> bare function
         frozendesc = self.rtyper.annotator.bookkeeper.getdesc(pbc)
         return self.convert_desc(frozendesc)
 
@@ -448,7 +449,7 @@
 
     def convert_desc(self, frozendesc):
         if (self.access_set is not None and
-            frozendesc not in self.access_set.descs):
+                frozendesc not in self.access_set.descs):
             raise TyperError("not found in PBC access set: %r" % (frozendesc,))
         try:
             return self.pbc_cache[frozendesc]
@@ -523,7 +524,10 @@
 
         im_selves = []
         for desc in s_pbc.descriptions:
-            assert desc.funcdesc is self.funcdesc, "You can't mix a set of methods on a frozen PBC in RPython that are different underlaying functions"
+            if desc.funcdesc is not self.funcdesc:
+                raise TyperError(
+                    "You can't mix a set of methods on a frozen PBC in "
+                    "RPython that are different underlying functions")
             im_selves.append(desc.frozendesc)
 
         self.s_im_self = annmodel.SomePBC(im_selves)

From noreply at buildbot.pypy.org  Sat Apr 18 18:57:36 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Sat, 18 Apr 2015 18:57:36 +0200 (CEST)
Subject: [pypy-commit] pypy callfamily: Create LLCallTable class
Message-ID: <20150418165736.D2C2C1C0484@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: callfamily
Changeset: r76826:40ce9a36c547
Date: 2015-04-18 04:41 +0100
http://bitbucket.org/pypy/pypy/changeset/40ce9a36c547/

Log:	Create LLCallTable class

diff --git a/rpython/rtyper/lltypesystem/rpbc.py b/rpython/rtyper/lltypesystem/rpbc.py
--- a/rpython/rtyper/lltypesystem/rpbc.py
+++ b/rpython/rtyper/lltypesystem/rpbc.py
@@ -105,9 +105,8 @@
         self.rtyper = rtyper
         self.s_pbc = s_pbc
         self.callfamily = s_pbc.any_description().getcallfamily()
-        concretetable, uniquerows = get_concrete_calltable(self.rtyper,
-                                                           self.callfamily)
-        assert len(uniquerows) == 1
+        llct = get_concrete_calltable(self.rtyper, self.callfamily)
+        assert len(llct.uniquerows) == 1
         self.lowleveltype = Char
         self.pointer_repr = FunctionsPBCRepr(rtyper, s_pbc)
         self._conversion_tables = {}
diff --git a/rpython/rtyper/rpbc.py b/rpython/rtyper/rpbc.py
--- a/rpython/rtyper/rpbc.py
+++ b/rpython/rtyper/rpbc.py
@@ -17,8 +17,8 @@
 def small_cand(rtyper, s_pbc):
     if 1 < len(s_pbc.descriptions) < rtyper.getconfig().translation.withsmallfuncsets:
         callfamily = s_pbc.any_description().getcallfamily()
-        concretetable, uniquerows = get_concrete_calltable(rtyper, callfamily)
-        if (len(uniquerows) == 1 and
+        llct = get_concrete_calltable(rtyper, callfamily)
+        if (len(llct.uniquerows) == 1 and
                 (not s_pbc.subset_of or small_cand(rtyper, s_pbc.subset_of))):
             return True
     return False
@@ -65,12 +65,18 @@
 class ConcreteCallTableRow(dict):
     """A row in a concrete call table."""
 
+class LLCallTable(object):
+    """A call table of a call family with low-level functions."""
+    def __init__(self, table, uniquerows):
+        self.table = table  # (shape,index): row, maybe with duplicates
+        self.uniquerows = uniquerows  # list of rows, without duplicates
+
 def build_concrete_calltable(rtyper, callfamily):
     """Build a complete call table of a call family
     with concrete low-level function objs.
     """
-    concretetable = {}   # (shape,index): row, maybe with duplicates
-    uniquerows = []      # list of rows, without duplicates
+    concretetable = {}
+    uniquerows = []
 
     def lookuprow(row):
         # a 'matching' row is one that has the same llfn, expect
@@ -133,7 +139,7 @@
         for finalindex, row in enumerate(uniquerows):
             row.attrname = 'variant%d' % finalindex
 
-    return concretetable, uniquerows
+    return LLCallTable(concretetable, uniquerows)
 
 def get_concrete_calltable(rtyper, callfamily):
     """Get a complete call table of a call family
@@ -143,14 +149,14 @@
     try:
         cached = rtyper.concrete_calltables[callfamily]
     except KeyError:
-        concretetable, uniquerows = build_concrete_calltable(rtyper, callfamily)
-        cached = concretetable, uniquerows, callfamily.total_calltable_size
+        llct = build_concrete_calltable(rtyper, callfamily)
+        cached = llct, callfamily.total_calltable_size
         rtyper.concrete_calltables[callfamily] = cached
     else:
-        concretetable, uniquerows, oldsize = cached
+        llct, oldsize = cached
         if oldsize != callfamily.total_calltable_size:
             raise TyperError("call table was unexpectedly extended")
-    return concretetable, uniquerows
+    return llct
 
 
 class AbstractFunctionsPBCRepr(CanBeNull, Repr):
@@ -164,12 +170,11 @@
             # a single function
             self.lowleveltype = lltype.Void
         else:
-            concretetable, uniquerows = get_concrete_calltable(self.rtyper,
-                                                               self.callfamily)
-            self.concretetable = concretetable
-            self.uniquerows = uniquerows
-            if len(uniquerows) == 1:
-                row = uniquerows[0]
+            llct = get_concrete_calltable(self.rtyper, self.callfamily)
+            self.concretetable = llct.table
+            self.uniquerows = llct.uniquerows
+            if len(llct.uniquerows) == 1:
+                row = llct.uniquerows[0]
                 self.lowleveltype = row.fntype
             else:
                 # several functions, each with several specialized variants.

From noreply at buildbot.pypy.org  Sat Apr 18 18:57:37 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Sat, 18 Apr 2015 18:57:37 +0200 (CEST)
Subject: [pypy-commit] pypy callfamily: turn nested functions into methods
	of LLCT
Message-ID: <20150418165737.ED82C1C0484@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: callfamily
Changeset: r76827:f16f1e10ab81
Date: 2015-04-18 17:42 +0100
http://bitbucket.org/pypy/pypy/changeset/f16f1e10ab81/

Log:	turn nested functions into methods of LLCT

diff --git a/rpython/rtyper/rpbc.py b/rpython/rtyper/rpbc.py
--- a/rpython/rtyper/rpbc.py
+++ b/rpython/rtyper/rpbc.py
@@ -71,17 +71,10 @@
         self.table = table  # (shape,index): row, maybe with duplicates
         self.uniquerows = uniquerows  # list of rows, without duplicates
 
-def build_concrete_calltable(rtyper, callfamily):
-    """Build a complete call table of a call family
-    with concrete low-level function objs.
-    """
-    concretetable = {}
-    uniquerows = []
-
-    def lookuprow(row):
-        # a 'matching' row is one that has the same llfn, expect
-        # that it may have more or less 'holes'
-        for existingindex, existingrow in enumerate(uniquerows):
+    def lookup(self, row):
+        """A 'matching' row is one that has the same llfn, except
+        that it may have more or less 'holes'."""
+        for existingindex, existingrow in enumerate(self.uniquerows):
             if row.fntype != existingrow.fntype:
                 continue   # not the same pointer type, cannot match
             for funcdesc, llfn in row.items():
@@ -99,18 +92,28 @@
                     return existingindex, merged
         raise LookupError
 
-    def addrow(row):
-        # add a row to the table, potentially merging it with an existing row
+    def add(self, row):
+        """Add a row to the table, potentially merging it with an existing row
+        """
         try:
-            index, merged = lookuprow(row)
+            index, merged = self.lookup(row)
         except LookupError:
-            uniquerows.append(row)   # new row
+            self.uniquerows.append(row)   # new row
         else:
-            if merged == uniquerows[index]:
+            if merged == self.uniquerows[index]:
                 pass    # already exactly in the table
             else:
-                del uniquerows[index]
-                addrow(merged)   # add the potentially larger merged row
+                del self.uniquerows[index]
+                self.add(merged)   # add the potentially larger merged row
+
+
+def build_concrete_calltable(rtyper, callfamily):
+    """Build a complete call table of a call family
+    with concrete low-level function objs.
+    """
+    concretetable = {}
+    uniquerows = []
+    llct = LLCallTable(concretetable, uniquerows)
 
     concreterows = {}
     for shape, rows in callfamily.calltables.items():
@@ -125,21 +128,21 @@
             concreterows[shape, index] = concreterow
 
     for row in concreterows.values():
-        addrow(row)
+        llct.add(row)
 
     for (shape, index), row in concreterows.items():
-        existingindex, biggerrow = lookuprow(row)
-        row = uniquerows[existingindex]
-        assert biggerrow == row   # otherwise, addrow() is broken
-        concretetable[shape, index] = row
+        existingindex, biggerrow = llct.lookup(row)
+        row = llct.uniquerows[existingindex]
+        assert biggerrow == row
+        llct.table[shape, index] = row
 
-    if len(uniquerows) == 1:
-        uniquerows[0].attrname = None
+    if len(llct.uniquerows) == 1:
+        llct.uniquerows[0].attrname = None
     else:
-        for finalindex, row in enumerate(uniquerows):
+        for finalindex, row in enumerate(llct.uniquerows):
             row.attrname = 'variant%d' % finalindex
 
-    return LLCallTable(concretetable, uniquerows)
+    return llct
 
 def get_concrete_calltable(rtyper, callfamily):
     """Get a complete call table of a call family

From noreply at buildbot.pypy.org  Sat Apr 18 19:15:46 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 18 Apr 2015 19:15:46 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: now, some mismatches show up lazily
Message-ID: <20150418171546.D39DA1C03F7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1767:fe0fdb4f6ce3
Date: 2015-04-18 19:16 +0200
http://bitbucket.org/cffi/cffi/changeset/fe0fdb4f6ce3/

Log:	now, some mismatches show up lazily

diff --git a/new/test_verify1.py b/new/test_verify1.py
--- a/new/test_verify1.py
+++ b/new/test_verify1.py
@@ -472,7 +472,8 @@
     ffi.cdef("struct foo_s { %s x; ...; };" % typename)
     try:
         ffi.verify("struct foo_s { %s x; };" % real)
-    except VerificationError:
+        ffi.new("struct foo_s *")    # because some mismatches show up lazily
+    except (VerificationError, ffi.error):
         if not expect_mismatch:
             if testing_by_size and typename != real:
                 print("ignoring mismatch between %s* and %s* even though "

From noreply at buildbot.pypy.org  Sat Apr 18 19:26:26 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 18 Apr 2015 19:26:26 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Add/remove files
Message-ID: <20150418172626.C19AB1C0484@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1768:4954444df617
Date: 2015-04-18 19:27 +0200
http://bitbucket.org/cffi/cffi/changeset/4954444df617/

Log:	Add/remove files

diff --git a/new/bsdopendirtype.py b/new/bsdopendirtype.py
new file mode 100644
--- /dev/null
+++ b/new/bsdopendirtype.py
@@ -0,0 +1,48 @@
+from _bsdopendirtype import ffi, lib
+
+
+def _posix_error():
+    raise OSError(ffi.errno, os.strerror(ffi.errno))
+
+_dtype_to_smode = {
+    lib.DT_BLK:  0o060000,
+    lib.DT_CHR:  0o020000,
+    lib.DT_DIR:  0o040000,
+    lib.DT_FIFO: 0o010000,
+    lib.DT_LNK:  0o120000,
+    lib.DT_REG:  0o100000,
+    lib.DT_SOCK: 0o140000,
+}
+
+def opendir(dir):
+    if len(dir) == 0:
+        dir = '.'
+    dirname = dir
+    if not dirname.endswith('/'):
+        dirname += '/'
+    dirp = lib.opendir(dir)
+    if dirp == ffi.NULL:
+        raise _posix_error()
+    try:
+        while True:
+            ffi.errno = 0
+            dirent = lib.readdir(dirp)
+            if dirent == ffi.NULL:
+                if ffi.errno != 0:
+                    raise _posix_error()
+                return
+            name = ffi.string(dirent.d_name)
+            if name == '.' or name == '..':
+                continue
+            name = dirname + name
+            try:
+                smode = _dtype_to_smode[dirent.d_type]
+            except KeyError:
+                smode = os.lstat(name).st_mode
+            yield name, smode
+    finally:
+        lib.closedir(dirp)
+
+if __name__ == '__main__':
+    for name, smode in opendir('/tmp'):
+        print hex(smode), name
diff --git a/new/bsdopendirtype_build.py b/new/bsdopendirtype_build.py
new file mode 100644
--- /dev/null
+++ b/new/bsdopendirtype_build.py
@@ -0,0 +1,21 @@
+from cffi1 import FFI
+import recompiler
+
+ffi = FFI()
+ffi.cdef("""
+    typedef ... DIR;
+    struct dirent {
+        unsigned char d_type;   /* type of file */
+        char d_name[];          /* filename */
+        ...;
+    };
+    DIR *opendir(const char *name);
+    int closedir(DIR *dirp);
+    struct dirent *readdir(DIR *dirp);
+    static const int DT_BLK, DT_CHR, DT_DIR, DT_FIFO, DT_LNK, DT_REG, DT_SOCK;
+""")
+
+recompiler.recompile(ffi, "_bsdopendirtype", """
+    #include 
+    #include 
+""")
diff --git a/new/udir.py b/new/udir.py
deleted file mode 100644
--- a/new/udir.py
+++ /dev/null
@@ -1,3 +0,0 @@
-import py
-
-udir = py.path.local.make_numbered_dir(prefix = 'ffi-')

From noreply at buildbot.pypy.org  Sat Apr 18 21:09:31 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 18 Apr 2015 21:09:31 +0200 (CEST)
Subject: [pypy-commit] pypy default: Explain this assertion error. Maybe it
 would be better to simply ignore
Message-ID: <20150418190931.6BC461C03F7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76828:b1dac6be879a
Date: 2015-04-18 21:09 +0200
http://bitbucket.org/pypy/pypy/changeset/b1dac6be879a/

Log:	Explain this assertion error. Maybe it would be better to simply
	ignore the operation if vinfo is None, but I feel that this really
	occurs only in case of mistake.

diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -1972,7 +1972,10 @@
     def rewrite_op_jit_force_virtualizable(self, op):
         # this one is for virtualizables
         vinfo = self.get_vinfo(op.args[0])
-        assert vinfo is not None
+        assert vinfo is not None, (
+            "%r is a class with _virtualizable_, but no jitdriver was found"
+            " with a 'virtualizable' argument naming a variable of that class"
+            % op.args[0].concretetype)
         self.vable_flags[op.args[0]] = op.args[2].value
         return []
 

From noreply at buildbot.pypy.org  Sat Apr 18 21:09:58 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Sat, 18 Apr 2015 21:09:58 +0200 (CEST)
Subject: [pypy-commit] pypy callfamily: Merge
 rpython.rtyper.lltypesystem.rpbc into rpython.rtyper.rpbc
Message-ID: <20150418190958.099EC1C03F7@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: callfamily
Changeset: r76829:19ac7c5e6243
Date: 2015-04-18 20:10 +0100
http://bitbucket.org/pypy/pypy/changeset/19ac7c5e6243/

Log:	Merge rpython.rtyper.lltypesystem.rpbc into rpython.rtyper.rpbc

diff --git a/rpython/rtyper/lltypesystem/rpbc.py b/rpython/rtyper/lltypesystem/rpbc.py
deleted file mode 100644
--- a/rpython/rtyper/lltypesystem/rpbc.py
+++ /dev/null
@@ -1,302 +0,0 @@
-import types
-
-from rpython.annotator import description
-from rpython.rlib.debug import ll_assert
-from rpython.rlib.unroll import unrolling_iterable
-from rpython.rtyper import callparse
-from rpython.rtyper.lltypesystem import llmemory
-from rpython.rtyper.lltypesystem.lltype import (typeOf, Void, ForwardReference,
-    Struct, Bool, Char, Ptr, malloc, nullptr, Array, Signed)
-from rpython.rtyper.rmodel import Repr, inputconst
-from rpython.rtyper.rpbc import (
-    AbstractMultipleFrozenPBCRepr,
-    AbstractFunctionsPBCRepr, AbstractMultipleUnrelatedFrozenPBCRepr,
-    SingleFrozenPBCRepr, get_concrete_calltable)
-from rpython.rtyper.typesystem import getfunctionptr
-from rpython.tool.pairtype import pairtype
-
-
-class MultipleFrozenPBCRepr(AbstractMultipleFrozenPBCRepr):
-    """Representation selected for multiple non-callable pre-built constants."""
-    def __init__(self, rtyper, access_set):
-        self.rtyper = rtyper
-        self.access_set = access_set
-        self.pbc_type = ForwardReference()
-        self.lowleveltype = Ptr(self.pbc_type)
-        self.pbc_cache = {}
-
-    def _setup_repr(self):
-        llfields = self._setup_repr_fields()
-        kwds = {'hints': {'immutable': True}}
-        self.pbc_type.become(Struct('pbc', *llfields, **kwds))
-
-    def create_instance(self):
-        return malloc(self.pbc_type, immortal=True)
-
-    def null_instance(self):
-        return nullptr(self.pbc_type)
-
-    def getfield(self, vpbc, attr, llops):
-        mangled_name, r_value = self.fieldmap[attr]
-        cmangledname = inputconst(Void, mangled_name)
-        return llops.genop('getfield', [vpbc, cmangledname],
-                           resulttype=r_value)
-
-
-class MultipleUnrelatedFrozenPBCRepr(AbstractMultipleUnrelatedFrozenPBCRepr):
-    """Representation selected for multiple non-callable pre-built constants
-    with no common access set."""
-
-    lowleveltype = llmemory.Address
-    EMPTY = Struct('pbc', hints={'immutable': True})
-
-    def convert_pbc(self, pbcptr):
-        return llmemory.fakeaddress(pbcptr)
-
-    def create_instance(self):
-        return malloc(self.EMPTY, immortal=True)
-
-    def null_instance(self):
-        return llmemory.Address._defl()
-
-
-class __extend__(pairtype(MultipleUnrelatedFrozenPBCRepr,
-                          MultipleUnrelatedFrozenPBCRepr),
-                 pairtype(MultipleUnrelatedFrozenPBCRepr,
-                          SingleFrozenPBCRepr),
-                 pairtype(SingleFrozenPBCRepr,
-                          MultipleUnrelatedFrozenPBCRepr)):
-    def rtype_is_((robj1, robj2), hop):
-        if isinstance(robj1, MultipleUnrelatedFrozenPBCRepr):
-            r = robj1
-        else:
-            r = robj2
-        vlist = hop.inputargs(r, r)
-        return hop.genop('adr_eq', vlist, resulttype=Bool)
-
-
-class __extend__(pairtype(MultipleFrozenPBCRepr,
-                          MultipleUnrelatedFrozenPBCRepr)):
-    def convert_from_to((robj1, robj2), v, llops):
-        return llops.genop('cast_ptr_to_adr', [v], resulttype=llmemory.Address)
-
-
-# ____________________________________________________________
-
-class FunctionsPBCRepr(AbstractFunctionsPBCRepr):
-    """Representation selected for a PBC of function(s)."""
-
-    def setup_specfunc(self):
-        fields = []
-        for row in self.uniquerows:
-            fields.append((row.attrname, row.fntype))
-        kwds = {'hints': {'immutable': True}}
-        return Ptr(Struct('specfunc', *fields, **kwds))
-
-    def create_specfunc(self):
-        return malloc(self.lowleveltype.TO, immortal=True)
-
-    def get_specfunc_row(self, llop, v, c_rowname, resulttype):
-        return llop.genop('getfield', [v, c_rowname], resulttype=resulttype)
-
-
-class SmallFunctionSetPBCRepr(Repr):
-    def __init__(self, rtyper, s_pbc):
-        self.rtyper = rtyper
-        self.s_pbc = s_pbc
-        self.callfamily = s_pbc.any_description().getcallfamily()
-        llct = get_concrete_calltable(self.rtyper, self.callfamily)
-        assert len(llct.uniquerows) == 1
-        self.lowleveltype = Char
-        self.pointer_repr = FunctionsPBCRepr(rtyper, s_pbc)
-        self._conversion_tables = {}
-        self._compression_function = None
-        self._dispatch_cache = {}
-
-    def _setup_repr(self):
-        if self.s_pbc.subset_of:
-            assert self.s_pbc.can_be_None == self.s_pbc.subset_of.can_be_None
-            r = self.rtyper.getrepr(self.s_pbc.subset_of)
-            if r is not self:
-                r.setup()
-                self.descriptions = r.descriptions
-                self.c_pointer_table = r.c_pointer_table
-                return
-        self.descriptions = list(self.s_pbc.descriptions)
-        if self.s_pbc.can_be_None:
-            self.descriptions.insert(0, None)
-        POINTER_TABLE = Array(self.pointer_repr.lowleveltype,
-                              hints={'nolength': True})
-        pointer_table = malloc(POINTER_TABLE, len(self.descriptions),
-                               immortal=True)
-        for i, desc in enumerate(self.descriptions):
-            if desc is not None:
-                pointer_table[i] = self.pointer_repr.convert_desc(desc)
-            else:
-                pointer_table[i] = self.pointer_repr.convert_const(None)
-        self.c_pointer_table = inputconst(Ptr(POINTER_TABLE), pointer_table)
-
-    def get_s_callable(self):
-        return self.s_pbc
-
-    def get_r_implfunc(self):
-        return self, 0
-
-    def get_s_signatures(self, shape):
-        funcdesc = self.s_pbc.any_description()
-        return funcdesc.get_s_signatures(shape)
-
-    def convert_desc(self, funcdesc):
-        return chr(self.descriptions.index(funcdesc))
-
-    def convert_const(self, value):
-        if isinstance(value, types.MethodType) and value.im_self is None:
-            value = value.im_func   # unbound method -> bare function
-        if value is None:
-            return chr(0)
-        funcdesc = self.rtyper.annotator.bookkeeper.getdesc(value)
-        return self.convert_desc(funcdesc)
-
-    def rtype_simple_call(self, hop):
-        return self.call(hop)
-
-    def rtype_call_args(self, hop):
-        return self.call(hop)
-
-    def dispatcher(self, shape, index, argtypes, resulttype):
-        key = shape, index, tuple(argtypes), resulttype
-        if key in self._dispatch_cache:
-            return self._dispatch_cache[key]
-        from rpython.translator.unsimplify import varoftype
-        from rpython.flowspace.model import FunctionGraph, Link, Block, SpaceOperation
-        inputargs = [varoftype(t) for t in [Char] + argtypes]
-        startblock = Block(inputargs)
-        startblock.exitswitch = inputargs[0]
-        graph = FunctionGraph("dispatcher", startblock, varoftype(resulttype))
-        row_of_graphs = self.callfamily.calltables[shape][index]
-        links = []
-        descs = list(self.s_pbc.descriptions)
-        if self.s_pbc.can_be_None:
-            descs.insert(0, None)
-        for desc in descs:
-            if desc is None:
-                continue
-            args_v = [varoftype(t) for t in argtypes]
-            b = Block(args_v)
-            llfn = self.rtyper.getcallable(row_of_graphs[desc])
-            v_fn = inputconst(typeOf(llfn), llfn)
-            v_result = varoftype(resulttype)
-            b.operations.append(
-                SpaceOperation("direct_call", [v_fn] + args_v, v_result))
-            b.closeblock(Link([v_result], graph.returnblock))
-            i = self.descriptions.index(desc)
-            links.append(Link(inputargs[1:], b, chr(i)))
-            links[-1].llexitcase = chr(i)
-        startblock.closeblock(*links)
-        self.rtyper.annotator.translator.graphs.append(graph)
-        ll_ret = getfunctionptr(graph)
-        #FTYPE = FuncType
-        c_ret = self._dispatch_cache[key] = inputconst(typeOf(ll_ret), ll_ret)
-        return c_ret
-
-    def call(self, hop):
-        bk = self.rtyper.annotator.bookkeeper
-        args = hop.spaceop.build_args(hop.args_s[1:])
-        s_pbc = hop.args_s[0]   # possibly more precise than self.s_pbc
-        descs = list(s_pbc.descriptions)
-        vfcs = description.FunctionDesc.variant_for_call_site
-        shape, index = vfcs(bk, self.callfamily, descs, args, hop.spaceop)
-        row_of_graphs = self.callfamily.calltables[shape][index]
-        anygraph = row_of_graphs.itervalues().next()  # pick any witness
-        vlist = [hop.inputarg(self, arg=0)]
-        vlist += callparse.callparse(self.rtyper, anygraph, hop)
-        rresult = callparse.getrresult(self.rtyper, anygraph)
-        hop.exception_is_here()
-        v_dispatcher = self.dispatcher(shape, index, [v.concretetype for v in vlist[1:]], rresult.lowleveltype)
-        v_result = hop.genop('direct_call', [v_dispatcher] + vlist,
-                             resulttype=rresult)
-        return hop.llops.convertvar(v_result, rresult, hop.r_result)
-
-    def rtype_bool(self, hop):
-        if not self.s_pbc.can_be_None:
-            return inputconst(Bool, True)
-        else:
-            v1, = hop.inputargs(self)
-            return hop.genop('char_ne', [v1, inputconst(Char, '\000')],
-                         resulttype=Bool)
-
-
-class __extend__(pairtype(SmallFunctionSetPBCRepr, FunctionsPBCRepr)):
-    def convert_from_to((r_set, r_ptr), v, llops):
-        if r_ptr.lowleveltype is Void:
-            return inputconst(Void, None)
-        else:
-            assert v.concretetype is Char
-            v_int = llops.genop('cast_char_to_int', [v],
-                                resulttype=Signed)
-            return llops.genop('getarrayitem', [r_set.c_pointer_table, v_int],
-                               resulttype=r_ptr.lowleveltype)
-
-
-def compression_function(r_set):
-    if r_set._compression_function is None:
-        table = []
-        for i, p in enumerate(r_set.c_pointer_table.value):
-            table.append((chr(i), p))
-        last_c, last_p = table[-1]
-        unroll_table = unrolling_iterable(table[:-1])
-
-        def ll_compress(fnptr):
-            for c, p in unroll_table:
-                if fnptr == p:
-                    return c
-            else:
-                ll_assert(fnptr == last_p, "unexpected function pointer")
-                return last_c
-        r_set._compression_function = ll_compress
-    return r_set._compression_function
-
-
-class __extend__(pairtype(FunctionsPBCRepr, SmallFunctionSetPBCRepr)):
-    def convert_from_to((r_ptr, r_set), v, llops):
-        if r_ptr.lowleveltype is Void:
-            desc, = r_ptr.s_pbc.descriptions
-            return inputconst(Char, r_set.convert_desc(desc))
-        else:
-            ll_compress = compression_function(r_set)
-            return llops.gendirectcall(ll_compress, v)
-
-
-def conversion_table(r_from, r_to):
-    if r_to in r_from._conversion_tables:
-        return r_from._conversion_tables[r_to]
-    else:
-        t = malloc(Array(Char, hints={'nolength': True}),
-                   len(r_from.descriptions), immortal=True)
-        l = []
-        for i, d in enumerate(r_from.descriptions):
-            if d in r_to.descriptions:
-                j = r_to.descriptions.index(d)
-                l.append(j)
-                t[i] = chr(j)
-            else:
-                l.append(None)
-        if l == range(len(r_from.descriptions)):
-            r = None
-        else:
-            r = inputconst(Ptr(Array(Char, hints={'nolength': True})), t)
-        r_from._conversion_tables[r_to] = r
-        return r
-
-
-class __extend__(pairtype(SmallFunctionSetPBCRepr, SmallFunctionSetPBCRepr)):
-    def convert_from_to((r_from, r_to), v, llops):
-        c_table = conversion_table(r_from, r_to)
-        if c_table:
-            assert v.concretetype is Char
-            v_int = llops.genop('cast_char_to_int', [v],
-                                resulttype=Signed)
-            return llops.genop('getarrayitem', [c_table, v_int],
-                               resulttype=Char)
-        else:
-            return v
diff --git a/rpython/rtyper/rnone.py b/rpython/rtyper/rnone.py
--- a/rpython/rtyper/rnone.py
+++ b/rpython/rtyper/rnone.py
@@ -3,7 +3,7 @@
 from rpython.rtyper.rmodel import Repr, TyperError, inputconst
 from rpython.rtyper.lltypesystem.lltype import Void, Bool, Ptr, Char
 from rpython.rtyper.lltypesystem.llmemory import Address
-from rpython.rtyper.lltypesystem.rpbc import SmallFunctionSetPBCRepr
+from rpython.rtyper.rpbc import SmallFunctionSetPBCRepr
 from rpython.rtyper.annlowlevel import llstr
 from rpython.tool.pairtype import pairtype
 
diff --git a/rpython/rtyper/rpbc.py b/rpython/rtyper/rpbc.py
--- a/rpython/rtyper/rpbc.py
+++ b/rpython/rtyper/rpbc.py
@@ -5,13 +5,20 @@
     FunctionDesc, ClassDesc, MethodDesc, FrozenDesc, MethodOfFrozenDesc)
 from rpython.flowspace.model import Constant
 from rpython.annotator.argument import simple_args
+from rpython.rlib.debug import ll_assert
+from rpython.rlib.unroll import unrolling_iterable
 from rpython.rtyper import rclass, callparse
 from rpython.rtyper.rclass import CLASSTYPE, OBJECT_VTABLE, OBJECTPTR
 from rpython.rtyper.error import TyperError
-from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.lltypesystem import rffi
+from rpython.rtyper.lltypesystem import llmemory
+from rpython.rtyper.lltypesystem.lltype import (typeOf, Void, ForwardReference,
+    Struct, Bool, Char, Ptr, malloc, nullptr, Array, Signed)
 from rpython.rtyper.rmodel import (Repr, inputconst, CanBeNull, mangle,
     warning, impossible_repr)
+from rpython.rtyper.typesystem import getfunctionptr
 from rpython.tool.pairtype import pair, pairtype
+from rpython.translator.unsimplify import varoftype
 
 
 def small_cand(rtyper, s_pbc):
@@ -25,8 +32,6 @@
 
 class __extend__(annmodel.SomePBC):
     def rtyper_makerepr(self, rtyper):
-        from rpython.rtyper.lltypesystem.rpbc import (
-            FunctionsPBCRepr, SmallFunctionSetPBCRepr)
         kind = self.getKind()
         if issubclass(kind, FunctionDesc):
             sample = self.any_description()
@@ -124,7 +129,7 @@
                 concreterow[funcdesc] = llfn
             assert len(concreterow) > 0
             # 'llfn' should be the same for all graphs
-            concreterow.fntype = lltype.typeOf(llfn)
+            concreterow.fntype = typeOf(llfn)
             concreterows[shape, index] = concreterow
 
     for row in concreterows.values():
@@ -162,7 +167,7 @@
     return llct
 
 
-class AbstractFunctionsPBCRepr(CanBeNull, Repr):
+class FunctionsPBCRepr(CanBeNull, Repr):
     """Representation selected for a PBC of function(s)."""
 
     def __init__(self, rtyper, s_pbc):
@@ -171,7 +176,7 @@
         self.callfamily = s_pbc.any_description().getcallfamily()
         if len(s_pbc.descriptions) == 1 and not s_pbc.can_be_None:
             # a single function
-            self.lowleveltype = lltype.Void
+            self.lowleveltype = Void
         else:
             llct = get_concrete_calltable(self.rtyper, self.callfamily)
             self.concretetable = llct.table
@@ -186,6 +191,19 @@
                 self.lowleveltype = self.setup_specfunc()
         self.funccache = {}
 
+    def setup_specfunc(self):
+        fields = []
+        for row in self.uniquerows:
+            fields.append((row.attrname, row.fntype))
+        kwds = {'hints': {'immutable': True}}
+        return Ptr(Struct('specfunc', *fields, **kwds))
+
+    def create_specfunc(self):
+        return malloc(self.lowleveltype.TO, immortal=True)
+
+    def get_specfunc_row(self, llop, v, c_rowname, resulttype):
+        return llop.genop('getfield', [v, c_rowname], resulttype=resulttype)
+
     def get_s_callable(self):
         return self.s_pbc
 
@@ -202,7 +220,7 @@
             return self.funccache[funcdesc]
         except KeyError:
             pass
-        if self.lowleveltype is lltype.Void:
+        if self.lowleveltype is Void:
             result = None
         else:
             llfns = {}
@@ -238,7 +256,7 @@
             value = value.im_func  # unbound method -> bare function
         elif isinstance(value, staticmethod):
             value = value.__get__(42)  # hackish, get the function wrapped by staticmethod
-        if self.lowleveltype is lltype.Void:
+        if self.lowleveltype is Void:
             return None
         if value is None:
             null = self.rtyper.type_system.null_callable(self.lowleveltype)
@@ -252,27 +270,27 @@
         'index' and 'shape' tells which of its items we are interested in.
         """
         assert v.concretetype == self.lowleveltype
-        if self.lowleveltype is lltype.Void:
+        if self.lowleveltype is Void:
             assert len(self.s_pbc.descriptions) == 1
                                       # lowleveltype wouldn't be Void otherwise
             funcdesc, = self.s_pbc.descriptions
             row_of_one_graph = self.callfamily.calltables[shape][index]
             graph = row_of_one_graph[funcdesc]
             llfn = self.rtyper.getcallable(graph)
-            return inputconst(lltype.typeOf(llfn), llfn)
+            return inputconst(typeOf(llfn), llfn)
         elif len(self.uniquerows) == 1:
             return v
         else:
             # 'v' is a Struct pointer, read the corresponding field
             row = self.concretetable[shape, index]
-            cname = inputconst(lltype.Void, row.attrname)
+            cname = inputconst(Void, row.attrname)
             return self.get_specfunc_row(llop, v, cname, row.fntype)
 
     def get_unique_llfn(self):
         # try to build a unique low-level function.  Avoid to use
         # whenever possible!  Doesn't work with specialization, multiple
         # different call sites, etc.
-        if self.lowleveltype is not lltype.Void:
+        if self.lowleveltype is not Void:
             raise TyperError("cannot pass multiple functions here")
         assert len(self.s_pbc.descriptions) == 1
                                   # lowleveltype wouldn't be Void otherwise
@@ -294,7 +312,7 @@
         if graphs != [graph] * len(graphs):
             raise TyperError("cannot pass a specialized function here")
         llfn = self.rtyper.getcallable(graph)
-        return inputconst(lltype.typeOf(llfn), llfn)
+        return inputconst(typeOf(llfn), llfn)
 
     def get_concrete_llfn(self, s_pbc, args_s, op):
         bk = self.rtyper.annotator.bookkeeper
@@ -306,7 +324,7 @@
         row_of_one_graph = self.callfamily.calltables[shape][index]
         graph = row_of_one_graph[funcdesc]
         llfn = self.rtyper.getcallable(graph)
-        return inputconst(lltype.typeOf(llfn), llfn)
+        return inputconst(typeOf(llfn), llfn)
 
     def rtype_simple_call(self, hop):
         return self.call(hop)
@@ -332,28 +350,227 @@
         if isinstance(vlist[0], Constant):
             v = hop.genop('direct_call', vlist, resulttype=rresult)
         else:
-            vlist.append(hop.inputconst(lltype.Void, row_of_graphs.values()))
+            vlist.append(hop.inputconst(Void, row_of_graphs.values()))
             v = hop.genop('indirect_call', vlist, resulttype=rresult)
         if hop.r_result is impossible_repr:
             return None      # see test_always_raising_methods
         else:
             return hop.llops.convertvar(v, rresult, hop.r_result)
 
-class __extend__(pairtype(AbstractFunctionsPBCRepr, AbstractFunctionsPBCRepr)):
+class __extend__(pairtype(FunctionsPBCRepr, FunctionsPBCRepr)):
     def convert_from_to((r_fpbc1, r_fpbc2), v, llops):
         # this check makes sense because both source and dest repr are FunctionsPBCRepr
         if r_fpbc1.lowleveltype == r_fpbc2.lowleveltype:
             return v
-        if r_fpbc1.lowleveltype is lltype.Void:
+        if r_fpbc1.lowleveltype is Void:
             return inputconst(r_fpbc2, r_fpbc1.s_pbc.const)
-        if r_fpbc2.lowleveltype is lltype.Void:
-            return inputconst(lltype.Void, None)
+        if r_fpbc2.lowleveltype is Void:
+            return inputconst(Void, None)
         return NotImplemented
 
 
+class SmallFunctionSetPBCRepr(Repr):
+    def __init__(self, rtyper, s_pbc):
+        self.rtyper = rtyper
+        self.s_pbc = s_pbc
+        self.callfamily = s_pbc.any_description().getcallfamily()
+        llct = get_concrete_calltable(self.rtyper, self.callfamily)
+        assert len(llct.uniquerows) == 1
+        self.lowleveltype = Char
+        self.pointer_repr = FunctionsPBCRepr(rtyper, s_pbc)
+        self._conversion_tables = {}
+        self._compression_function = None
+        self._dispatch_cache = {}
+
+    def _setup_repr(self):
+        if self.s_pbc.subset_of:
+            assert self.s_pbc.can_be_None == self.s_pbc.subset_of.can_be_None
+            r = self.rtyper.getrepr(self.s_pbc.subset_of)
+            if r is not self:
+                r.setup()
+                self.descriptions = r.descriptions
+                self.c_pointer_table = r.c_pointer_table
+                return
+        self.descriptions = list(self.s_pbc.descriptions)
+        if self.s_pbc.can_be_None:
+            self.descriptions.insert(0, None)
+        POINTER_TABLE = Array(self.pointer_repr.lowleveltype,
+                              hints={'nolength': True})
+        pointer_table = malloc(POINTER_TABLE, len(self.descriptions),
+                               immortal=True)
+        for i, desc in enumerate(self.descriptions):
+            if desc is not None:
+                pointer_table[i] = self.pointer_repr.convert_desc(desc)
+            else:
+                pointer_table[i] = self.pointer_repr.convert_const(None)
+        self.c_pointer_table = inputconst(Ptr(POINTER_TABLE), pointer_table)
+
+    def get_s_callable(self):
+        return self.s_pbc
+
+    def get_r_implfunc(self):
+        return self, 0
+
+    def get_s_signatures(self, shape):
+        funcdesc = self.s_pbc.any_description()
+        return funcdesc.get_s_signatures(shape)
+
+    def convert_desc(self, funcdesc):
+        return chr(self.descriptions.index(funcdesc))
+
+    def convert_const(self, value):
+        if isinstance(value, types.MethodType) and value.im_self is None:
+            value = value.im_func   # unbound method -> bare function
+        if value is None:
+            return chr(0)
+        funcdesc = self.rtyper.annotator.bookkeeper.getdesc(value)
+        return self.convert_desc(funcdesc)
+
+    def rtype_simple_call(self, hop):
+        return self.call(hop)
+
+    def rtype_call_args(self, hop):
+        return self.call(hop)
+
+    def dispatcher(self, shape, index, argtypes, resulttype):
+        key = shape, index, tuple(argtypes), resulttype
+        if key in self._dispatch_cache:
+            return self._dispatch_cache[key]
+        inputargs = [varoftype(t) for t in [Char] + argtypes]
+        startblock = Block(inputargs)
+        startblock.exitswitch = inputargs[0]
+        graph = FunctionGraph("dispatcher", startblock, varoftype(resulttype))
+        row_of_graphs = self.callfamily.calltables[shape][index]
+        links = []
+        descs = list(self.s_pbc.descriptions)
+        if self.s_pbc.can_be_None:
+            descs.insert(0, None)
+        for desc in descs:
+            if desc is None:
+                continue
+            args_v = [varoftype(t) for t in argtypes]
+            b = Block(args_v)
+            llfn = self.rtyper.getcallable(row_of_graphs[desc])
+            v_fn = inputconst(typeOf(llfn), llfn)
+            v_result = varoftype(resulttype)
+            b.operations.append(
+                SpaceOperation("direct_call", [v_fn] + args_v, v_result))
+            b.closeblock(Link([v_result], graph.returnblock))
+            i = self.descriptions.index(desc)
+            links.append(Link(inputargs[1:], b, chr(i)))
+            links[-1].llexitcase = chr(i)
+        startblock.closeblock(*links)
+        self.rtyper.annotator.translator.graphs.append(graph)
+        ll_ret = getfunctionptr(graph)
+        #FTYPE = FuncType
+        c_ret = self._dispatch_cache[key] = inputconst(typeOf(ll_ret), ll_ret)
+        return c_ret
+
+    def call(self, hop):
+        bk = self.rtyper.annotator.bookkeeper
+        args = hop.spaceop.build_args(hop.args_s[1:])
+        s_pbc = hop.args_s[0]   # possibly more precise than self.s_pbc
+        descs = list(s_pbc.descriptions)
+        vfcs = FunctionDesc.variant_for_call_site
+        shape, index = vfcs(bk, self.callfamily, descs, args, hop.spaceop)
+        row_of_graphs = self.callfamily.calltables[shape][index]
+        anygraph = row_of_graphs.itervalues().next()  # pick any witness
+        vlist = [hop.inputarg(self, arg=0)]
+        vlist += callparse.callparse(self.rtyper, anygraph, hop)
+        rresult = callparse.getrresult(self.rtyper, anygraph)
+        hop.exception_is_here()
+        v_dispatcher = self.dispatcher(shape, index,
+                [v.concretetype for v in vlist[1:]], rresult.lowleveltype)
+        v_result = hop.genop('direct_call', [v_dispatcher] + vlist,
+                             resulttype=rresult)
+        return hop.llops.convertvar(v_result, rresult, hop.r_result)
+
+    def rtype_bool(self, hop):
+        if not self.s_pbc.can_be_None:
+            return inputconst(Bool, True)
+        else:
+            v1, = hop.inputargs(self)
+            return hop.genop('char_ne', [v1, inputconst(Char, '\000')],
+                         resulttype=Bool)
+
+
+class __extend__(pairtype(SmallFunctionSetPBCRepr, FunctionsPBCRepr)):
+    def convert_from_to((r_set, r_ptr), v, llops):
+        if r_ptr.lowleveltype is Void:
+            return inputconst(Void, None)
+        else:
+            assert v.concretetype is Char
+            v_int = llops.genop('cast_char_to_int', [v],
+                                resulttype=Signed)
+            return llops.genop('getarrayitem', [r_set.c_pointer_table, v_int],
+                               resulttype=r_ptr.lowleveltype)
+
+
+def compression_function(r_set):
+    if r_set._compression_function is None:
+        table = []
+        for i, p in enumerate(r_set.c_pointer_table.value):
+            table.append((chr(i), p))
+        last_c, last_p = table[-1]
+        unroll_table = unrolling_iterable(table[:-1])
+
+        def ll_compress(fnptr):
+            for c, p in unroll_table:
+                if fnptr == p:
+                    return c
+            else:
+                ll_assert(fnptr == last_p, "unexpected function pointer")
+                return last_c
+        r_set._compression_function = ll_compress
+    return r_set._compression_function
+
+
+class __extend__(pairtype(FunctionsPBCRepr, SmallFunctionSetPBCRepr)):
+    def convert_from_to((r_ptr, r_set), v, llops):
+        if r_ptr.lowleveltype is Void:
+            desc, = r_ptr.s_pbc.descriptions
+            return inputconst(Char, r_set.convert_desc(desc))
+        else:
+            ll_compress = compression_function(r_set)
+            return llops.gendirectcall(ll_compress, v)
+
+
+def conversion_table(r_from, r_to):
+    if r_to in r_from._conversion_tables:
+        return r_from._conversion_tables[r_to]
+    else:
+        t = malloc(Array(Char, hints={'nolength': True}),
+                   len(r_from.descriptions), immortal=True)
+        l = []
+        for i, d in enumerate(r_from.descriptions):
+            if d in r_to.descriptions:
+                j = r_to.descriptions.index(d)
+                l.append(j)
+                t[i] = chr(j)
+            else:
+                l.append(None)
+        if l == range(len(r_from.descriptions)):
+            r = None
+        else:
+            r = inputconst(Ptr(Array(Char, hints={'nolength': True})), t)
+        r_from._conversion_tables[r_to] = r
+        return r
+
+
+class __extend__(pairtype(SmallFunctionSetPBCRepr, SmallFunctionSetPBCRepr)):
+    def convert_from_to((r_from, r_to), v, llops):
+        c_table = conversion_table(r_from, r_to)
+        if c_table:
+            assert v.concretetype is Char
+            v_int = llops.genop('cast_char_to_int', [v],
+                                resulttype=Signed)
+            return llops.genop('getarrayitem', [c_table, v_int],
+                               resulttype=Char)
+        else:
+            return v
+
+
 def getFrozenPBCRepr(rtyper, s_pbc):
-    from rpython.rtyper.lltypesystem.rpbc import (
-        MultipleUnrelatedFrozenPBCRepr, MultipleFrozenPBCRepr)
     descs = list(s_pbc.descriptions)
     assert len(descs) >= 1
     if len(descs) == 1 and not s_pbc.can_be_None:
@@ -380,7 +597,7 @@
 
 class SingleFrozenPBCRepr(Repr):
     """Representation selected for a single non-callable pre-built constant."""
-    lowleveltype = lltype.Void
+    lowleveltype = Void
 
     def __init__(self, frozendesc):
         self.frozendesc = frozendesc
@@ -405,9 +622,18 @@
         return self.getstr()
 
 
-class AbstractMultipleUnrelatedFrozenPBCRepr(CanBeNull, Repr):
+class MultipleFrozenPBCReprBase(CanBeNull, Repr):
+    def convert_const(self, pbc):
+        if pbc is None:
+            return self.null_instance()
+        frozendesc = self.rtyper.annotator.bookkeeper.getdesc(pbc)
+        return self.convert_desc(frozendesc)
+
+class MultipleUnrelatedFrozenPBCRepr(MultipleFrozenPBCReprBase):
     """For a SomePBC of frozen PBCs that have no common access set.
     The only possible operation on such a thing is comparison with 'is'."""
+    lowleveltype = llmemory.Address
+    EMPTY = Struct('pbc', hints={'immutable': True})
 
     def __init__(self, rtyper):
         self.rtyper = rtyper
@@ -418,7 +644,7 @@
             return self.converted_pbc_cache[frozendesc]
         except KeyError:
             r = self.rtyper.getrepr(annmodel.SomePBC([frozendesc]))
-            if r.lowleveltype is lltype.Void:
+            if r.lowleveltype is Void:
                 # must create a new empty structure, as a placeholder
                 pbc = self.create_instance()
             else:
@@ -427,20 +653,62 @@
             self.converted_pbc_cache[frozendesc] = convpbc
             return convpbc
 
-    def convert_const(self, pbc):
-        if pbc is None:
-            return self.null_instance()
-        frozendesc = self.rtyper.annotator.bookkeeper.getdesc(pbc)
-        return self.convert_desc(frozendesc)
+    def convert_pbc(self, pbcptr):
+        return llmemory.fakeaddress(pbcptr)
+
+    def create_instance(self):
+        return malloc(self.EMPTY, immortal=True)
+
+    def null_instance(self):
+        return llmemory.Address._defl()
 
     def rtype_getattr(_, hop):
         if not hop.s_result.is_constant():
             raise TyperError("getattr on a constant PBC returns a non-constant")
         return hop.inputconst(hop.r_result, hop.s_result.const)
 
-class AbstractMultipleFrozenPBCRepr(AbstractMultipleUnrelatedFrozenPBCRepr):
+class __extend__(pairtype(MultipleUnrelatedFrozenPBCRepr,
+                          MultipleUnrelatedFrozenPBCRepr),
+                 pairtype(MultipleUnrelatedFrozenPBCRepr,
+                          SingleFrozenPBCRepr),
+                 pairtype(SingleFrozenPBCRepr,
+                          MultipleUnrelatedFrozenPBCRepr)):
+    def rtype_is_((robj1, robj2), hop):
+        if isinstance(robj1, MultipleUnrelatedFrozenPBCRepr):
+            r = robj1
+        else:
+            r = robj2
+        vlist = hop.inputargs(r, r)
+        return hop.genop('adr_eq', vlist, resulttype=Bool)
+
+
+class MultipleFrozenPBCRepr(MultipleFrozenPBCReprBase):
     """For a SomePBC of frozen PBCs with a common attribute access set."""
 
+    def __init__(self, rtyper, access_set):
+        self.rtyper = rtyper
+        self.access_set = access_set
+        self.pbc_type = ForwardReference()
+        self.lowleveltype = Ptr(self.pbc_type)
+        self.pbc_cache = {}
+
+    def _setup_repr(self):
+        llfields = self._setup_repr_fields()
+        kwds = {'hints': {'immutable': True}}
+        self.pbc_type.become(Struct('pbc', *llfields, **kwds))
+
+    def create_instance(self):
+        return malloc(self.pbc_type, immortal=True)
+
+    def null_instance(self):
+        return nullptr(self.pbc_type)
+
+    def getfield(self, vpbc, attr, llops):
+        mangled_name, r_value = self.fieldmap[attr]
+        cmangledname = inputconst(Void, mangled_name)
+        return llops.genop('getfield', [vpbc, cmangledname],
+                           resulttype=r_value)
+
     def _setup_repr_fields(self):
         fields = []
         self.fieldmap = {}
@@ -466,7 +734,7 @@
             result = self.create_instance()
             self.pbc_cache[frozendesc] = result
             for attr, (mangled_name, r_value) in self.fieldmap.items():
-                if r_value.lowleveltype is lltype.Void:
+                if r_value.lowleveltype is Void:
                     continue
                 try:
                     thisattrvalue = frozendesc.attrcache[attr]
@@ -483,18 +751,23 @@
             return hop.inputconst(hop.r_result, hop.s_result.const)
 
         attr = hop.args_s[1].const
-        vpbc, vattr = hop.inputargs(self, lltype.Void)
+        vpbc, vattr = hop.inputargs(self, Void)
         v_res = self.getfield(vpbc, attr, hop.llops)
         mangled_name, r_res = self.fieldmap[attr]
         return hop.llops.convertvar(v_res, r_res, hop.r_result)
 
-class __extend__(pairtype(AbstractMultipleFrozenPBCRepr, AbstractMultipleFrozenPBCRepr)):
+class __extend__(pairtype(MultipleFrozenPBCRepr,
+                          MultipleUnrelatedFrozenPBCRepr)):
+    def convert_from_to((robj1, robj2), v, llops):
+        return llops.genop('cast_ptr_to_adr', [v], resulttype=llmemory.Address)
+
+class __extend__(pairtype(MultipleFrozenPBCRepr, MultipleFrozenPBCRepr)):
     def convert_from_to((r_pbc1, r_pbc2), v, llops):
         if r_pbc1.access_set == r_pbc2.access_set:
             return v
         return NotImplemented
 
-class __extend__(pairtype(SingleFrozenPBCRepr, AbstractMultipleFrozenPBCRepr)):
+class __extend__(pairtype(SingleFrozenPBCRepr, MultipleFrozenPBCRepr)):
     def convert_from_to((r_pbc1, r_pbc2), v, llops):
         frozendesc1 = r_pbc1.frozendesc
         access = frozendesc1.queryattrfamily()
@@ -504,10 +777,10 @@
             return Constant(value, lltype)
         return NotImplemented
 
-class __extend__(pairtype(AbstractMultipleUnrelatedFrozenPBCRepr,
+class __extend__(pairtype(MultipleFrozenPBCReprBase,
                           SingleFrozenPBCRepr)):
     def convert_from_to((r_pbc1, r_pbc2), v, llops):
-        return inputconst(lltype.Void, r_pbc2.frozendesc)
+        return inputconst(Void, r_pbc2.frozendesc)
 
 
 class MethodOfFrozenPBCRepr(Repr):
@@ -601,7 +874,7 @@
         #    raise TyperError("unsupported: variable of type "
         #                     "class-pointer or None")
         if s_pbc.is_constant():
-            self.lowleveltype = lltype.Void
+            self.lowleveltype = Void
         else:
             self.lowleveltype = self.getlowleveltype()
 
@@ -624,7 +897,7 @@
     def convert_desc(self, desc):
         if desc not in self.s_pbc.descriptions:
             raise TyperError("%r not in %r" % (desc, self))
-        if self.lowleveltype is lltype.Void:
+        if self.lowleveltype is Void:
             return None
         subclassdef = desc.getuniqueclassdef()
         r_subclass = rclass.getclassrepr(self.rtyper, subclassdef)
@@ -632,7 +905,7 @@
 
     def convert_const(self, cls):
         if cls is None:
-            if self.lowleveltype is lltype.Void:
+            if self.lowleveltype is Void:
                 return None
             else:
                 T = self.lowleveltype
@@ -649,12 +922,12 @@
             if attr == '__name__':
                 from rpython.rtyper.lltypesystem import rstr
                 class_repr = self.rtyper.rootclass_repr
-                vcls, vattr = hop.inputargs(class_repr, lltype.Void)
-                cname = inputconst(lltype.Void, 'name')
+                vcls, vattr = hop.inputargs(class_repr, Void)
+                cname = inputconst(Void, 'name')
                 return hop.genop('getfield', [vcls, cname],
-                                 resulttype = lltype.Ptr(rstr.STR))
+                                 resulttype = Ptr(rstr.STR))
             access_set, class_repr = self.get_access_set(attr)
-            vcls, vattr = hop.inputargs(class_repr, lltype.Void)
+            vcls, vattr = hop.inputargs(class_repr, Void)
             v_res = class_repr.getpbcfield(vcls, access_set, attr, hop.llops)
             s_res = access_set.s_value
             r_res = self.rtyper.getrepr(s_res)
@@ -683,7 +956,7 @@
 
         if len(self.s_pbc.descriptions) == 1:
             # instantiating a single class
-            if self.lowleveltype is not lltype.Void:
+            if self.lowleveltype is not Void:
                 assert 0, "XXX None-or-1-class instantation not implemented"
             assert isinstance(s_instance, annmodel.SomeInstance)
             classdef = s_instance.classdef
@@ -740,10 +1013,10 @@
             classdef = desc.getclassdef(None)
             assert hasattr(classdef, 'my_instantiate_graph')
             graphs.append(classdef.my_instantiate_graph)
-        c_graphs = hop.inputconst(lltype.Void, graphs)
+        c_graphs = hop.inputconst(Void, graphs)
         #
         # "my_instantiate = typeptr.instantiate"
-        c_name = hop.inputconst(lltype.Void, 'instantiate')
+        c_name = hop.inputconst(Void, 'instantiate')
         v_instantiate = hop.genop('getfield', [vtypeptr, c_name],
                                  resulttype=OBJECT_VTABLE.instantiate)
         # "my_instantiate()"
@@ -774,7 +1047,7 @@
         # turn a PBC of classes to a standard pointer-to-vtable class repr
         if r_clspbc.lowleveltype == r_cls.lowleveltype:
             return v
-        if r_clspbc.lowleveltype is lltype.Void:
+        if r_clspbc.lowleveltype is Void:
             return inputconst(r_cls, r_clspbc.s_pbc.const)
         # convert from ptr-to-object-vtable to ptr-to-more-precise-vtable
         return r_cls.fromclasstype(v, llops)
@@ -784,10 +1057,10 @@
         # this check makes sense because both source and dest repr are ClassesPBCRepr
         if r_clspbc1.lowleveltype == r_clspbc2.lowleveltype:
             return v
-        if r_clspbc1.lowleveltype is lltype.Void:
+        if r_clspbc1.lowleveltype is Void:
             return inputconst(r_clspbc2, r_clspbc1.s_pbc.const)
-        if r_clspbc2.lowleveltype is lltype.Void:
-            return inputconst(lltype.Void, r_clspbc2.s_pbc.const)
+        if r_clspbc2.lowleveltype is Void:
+            return inputconst(Void, r_clspbc2.s_pbc.const)
         return NotImplemented
 
 def adjust_shape(hop2, s_shape):
@@ -864,8 +1137,6 @@
         return self.redispatch_call(hop, call_args=True)
 
     def redispatch_call(self, hop, call_args):
-        from rpython.rtyper.lltypesystem.rpbc import (
-            FunctionsPBCRepr, SmallFunctionSetPBCRepr)
         r_class = self.r_im_self.rclass
         mangled_name, r_func = r_class.clsfields[self.methodname]
         assert isinstance(r_func, (FunctionsPBCRepr,
@@ -890,28 +1161,3 @@
 
         # now hop2 looks like simple_call(function, self, args...)
         return hop2.dispatch()
-
-# ____________________________________________________________
-
-def samesig(funcs):
-    import inspect
-    argspec = inspect.getargspec(funcs[0])
-    for func in funcs:
-        if inspect.getargspec(func) != argspec:
-            return False
-    return True
-
-# ____________________________________________________________
-
-def commonbase(classdefs):
-    result = classdefs[0]
-    for cdef in classdefs[1:]:
-        result = result.commonbase(cdef)
-        if result is None:
-            raise TyperError("no common base class in %r" % (classdefs,))
-    return result
-
-def allattributenames(classdef):
-    for cdef1 in classdef.getmro():
-        for attrname in cdef1.attrs:
-            yield cdef1, attrname

From noreply at buildbot.pypy.org  Sun Apr 19 10:34:17 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 19 Apr 2015 10:34:17 +0200 (CEST)
Subject: [pypy-commit] cffi default: Improve the test (passing) to check
 that comments are correctly ignored here
Message-ID: <20150419083417.F16161C02AD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r1769:ac7bf1ea3f1b
Date: 2015-04-19 09:50 +0200
http://bitbucket.org/cffi/cffi/changeset/ac7bf1ea3f1b/

Log:	Improve the test (passing) to check that comments are correctly
	ignored here

diff --git a/testing/test_verify.py b/testing/test_verify.py
--- a/testing/test_verify.py
+++ b/testing/test_verify.py
@@ -657,9 +657,9 @@
     # case the 'static' is completely ignored.
     ffi.cdef("static const int AA, BB, CC, DD;")
     lib = ffi.verify("#define AA 42\n"
-                     "#define BB (-43)\n"
-                     "#define CC (22*2)\n"
-                     "#define DD ((unsigned int)142)\n")
+                     "#define BB (-43)   // blah\n"
+                     "#define CC (22*2)  /* foobar */\n"
+                     "#define DD ((unsigned int)142)  /* foo\nbar */\n")
     assert lib.AA == 42
     assert lib.BB == -43
     assert lib.CC == 44

From noreply at buildbot.pypy.org  Sun Apr 19 10:34:19 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 19 Apr 2015 10:34:19 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Reimplement the distinction between
 "...; " and non-"...; " structures
Message-ID: <20150419083419.2B0D61C02AD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1770:9d55286c87b6
Date: 2015-04-19 10:34 +0200
http://bitbucket.org/cffi/cffi/changeset/9d55286c87b6/

Log:	Reimplement the distinction between "...;" and non-"...;" structures

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -272,6 +272,8 @@
 # include "wchar_helper.h"
 #endif
 
+static PyObject *FFIError;
+
 /************************************************************/
 
 static CTypeDescrObject *
@@ -3745,6 +3747,7 @@
 #define SF_GCC_LITTLE_ENDIAN  0x40
 
 #define SF_PACKED             0x08
+#define SF_STD_FIELD_POS      0x80
 
 static int complete_sflags(int sflags)
 {
@@ -3772,6 +3775,28 @@
     return sflags;
 }
 
+static int detect_custom_layout(CTypeDescrObject *ct, int sflags,
+                                Py_ssize_t cdef_value,
+                                Py_ssize_t compiler_value,
+                                const char *msg1, const char *txt,
+                                const char *msg2)
+{
+    if (compiler_value != cdef_value) {
+        if (sflags & SF_STD_FIELD_POS) {
+            PyErr_Format(FFIError,
+                         "%s: %s%s%s (cdef says %zd, but C compiler says %zd)."
+                         " fix it or use \"...;\" in the cdef for %s to "
+                         "make it flexible",
+                         ct->ct_name, msg1, txt, msg2,
+                         cdef_value, compiler_value,
+                         ct->ct_name);
+            return -1;
+        }
+        ct->ct_flags |= CT_CUSTOM_FIELD_POS;
+    }
+    return 0;
+}
+
 static PyObject *b_complete_struct_or_union(PyObject *self, PyObject *args)
 {
     CTypeDescrObject *ct;
@@ -3805,6 +3830,7 @@
                   "first arg must be a non-initialized struct or union ctype");
         return NULL;
     }
+    ct->ct_flags &= ~CT_CUSTOM_FIELD_POS;
 
     alignment = 1;
     boffset = 0;         /* this number is in *bits*, not bytes! */
@@ -3882,8 +3908,10 @@
             if (foffset >= 0) {
                 /* a forced field position: ignore the offset just computed,
                    except to know if we must set CT_CUSTOM_FIELD_POS */
-                if (boffset != foffset * 8)
-                    ct->ct_flags |= CT_CUSTOM_FIELD_POS;
+                if (detect_custom_layout(ct, sflags, boffset / 8, foffset,
+                                         "wrong offset for field '",
+                                         PyText_AS_UTF8(fname), "'") < 0)
+                    goto error;
                 boffset = foffset * 8;
             }
 
@@ -4061,14 +4089,28 @@
         if (totalsize == 0)
             totalsize = 1;
     }
-    else if (totalsize < boffsetmax) {
-        PyErr_Format(PyExc_TypeError,
-                     "%s cannot be of size %zd: there are fields at least "
-                     "up to %zd", ct->ct_name, totalsize, boffsetmax);
-        goto error;
-    }
+    else {
+        if (detect_custom_layout(ct, sflags, boffsetmax, totalsize,
+                                 "wrong total size", "", "") < 0)
+            goto error;
+        if (totalsize < boffsetmax) {
+            PyErr_Format(PyExc_TypeError,
+                         "%s cannot be of size %zd: there are fields at least "
+                         "up to %zd", ct->ct_name, totalsize, boffsetmax);
+            goto error;
+        }
+    }
+    if (totalalignment < 0) {
+        totalalignment = alignment;
+    }
+    else {
+        if (detect_custom_layout(ct, sflags, alignment, totalalignment,
+                                 "wrong total alignment", "", "") < 0)
+            goto error;
+    }
+
     ct->ct_size = totalsize;
-    ct->ct_length = totalalignment < 0 ? alignment : totalalignment;
+    ct->ct_length = totalalignment;
     ct->ct_stuff = interned_fields;
     ct->ct_flags &= ~CT_IS_OPAQUE;
 
@@ -4076,6 +4118,7 @@
     return Py_None;
 
  error:
+    ct->ct_extra = NULL;
     Py_DECREF(interned_fields);
     return NULL;
 }
diff --git a/new/cffi1_module.c b/new/cffi1_module.c
--- a/new/cffi1_module.c
+++ b/new/cffi1_module.c
@@ -1,4 +1,3 @@
-static PyObject *FFIError;
 
 #include "parse_c_type.c"
 #include "realize_c_type.c"
diff --git a/new/parse_c_type.h b/new/parse_c_type.h
--- a/new/parse_c_type.h
+++ b/new/parse_c_type.h
@@ -75,8 +75,11 @@
     int first_field_index;   // -> _cffi_fields array
     int num_fields;
 };
-#define CT_UNION            128
-#define CT_IS_OPAQUE        4096
+#define CT_UNION              128
+#define CT_IS_OPAQUE          4096
+#define CT_CUSTOM_FIELD_POS   32768
+/* ^^^ if not CUSTOM_FIELD_POS, complain if fields are not in the
+   "standard layout" and/or if some are missing */
 
 struct _cffi_field_s {
     const char *name;
diff --git a/new/realize_c_type.c b/new/realize_c_type.c
--- a/new/realize_c_type.c
+++ b/new/realize_c_type.c
@@ -451,14 +451,11 @@
                 return -1;
             }
 
-            if (ctf->ct_size != fld->field_size) {
-                PyErr_Format(FFIError,
-                             "%s field '%s' was declared in the cdef to be"
-                             " %zd bytes, but is actually %zd bytes",
-                             ct->ct_name, fld->name,
-                             ctf->ct_size, fld->field_size);
+            if (detect_custom_layout(ct, SF_STD_FIELD_POS,
+                                     ctf->ct_size, fld->field_size,
+                                     "wrong size for field '",
+                                     fld->name, "'") < 0)
                 return -1;
-            }
 
             f = Py_BuildValue("(sOin)", fld->name, ctf,
                               (int)-1, (Py_ssize_t)fld->field_offset);
@@ -469,19 +466,24 @@
             PyList_SET_ITEM(fields, i, f);
         }
 
-        PyObject *args = Py_BuildValue("(OOOnn)", ct, fields,
+        int sflags = (s->flags & CT_CUSTOM_FIELD_POS) ? 0 : SF_STD_FIELD_POS;
+
+        PyObject *args = Py_BuildValue("(OOOnni)", ct, fields,
                                        Py_None,
                                        (Py_ssize_t)s->size,
-                                       (Py_ssize_t)s->alignment);
+                                       (Py_ssize_t)s->alignment,
+                                       sflags);
         Py_DECREF(fields);
         if (args == NULL)
             return -1;
 
         ct->ct_extra = NULL;
         ct->ct_flags |= CT_IS_OPAQUE;
+        ct->ct_flags &= ~CT_CUSTOM_FIELD_POS;
         PyObject *res = b_complete_struct_or_union(NULL, args);
         ct->ct_flags &= ~CT_IS_OPAQUE;
         Py_DECREF(args);
+
         if (res == NULL) {
             ct->ct_extra = builder;
             return -1;
diff --git a/new/recompiler.py b/new/recompiler.py
--- a/new/recompiler.py
+++ b/new/recompiler.py
@@ -428,9 +428,12 @@
 
     def _generate_cpy_struct_ctx(self, tp, name):
         type_index = self._typesdict[tp]
-        flags = '0'
+        flags = []
+        if tp.partial:
+            flags.append('CT_CUSTOM_FIELD_POS')
         if isinstance(tp, model.UnionType):
-            flags = 'CT_UNION'
+            flags.append('CT_UNION')
+        flags = ('|'.join(flags)) or '0'
         if tp.fldtypes is not None:
             c_field = [name]
             for fldname, fldtype in zip(tp.fldnames, tp.fldtypes):
diff --git a/new/test_recompiler.py b/new/test_recompiler.py
--- a/new/test_recompiler.py
+++ b/new/test_recompiler.py
@@ -199,7 +199,7 @@
 
 def test_verify_struct():
     ffi = FFI()
-    ffi.cdef("""struct foo_s { int b; short a; };
+    ffi.cdef("""struct foo_s { int b; short a; ...; };
                 struct bar_s { struct foo_s *f; };""")
     lib = verify(ffi, 'test_verify_struct',
                  """struct foo_s { short a; int b; };
@@ -213,6 +213,16 @@
     q = ffi.new("struct bar_s *", {'f': p})
     assert q.f == p
 
+def test_verify_exact_field_offset():
+    ffi = FFI()
+    ffi.cdef("""struct foo_s { int b; short a; };""")
+    lib = verify(ffi, 'test_verify_exact_field_offset',
+                 """struct foo_s { short a; int b; };""")
+    e = py.test.raises(ffi.error, ffi.new, "struct foo_s *")    # lazily
+    assert str(e.value) == ("struct foo_s: wrong offset for field 'b' (cdef "
+                       'says 0, but C compiler says 4). fix it or use "...;" '
+                       "in the cdef for struct foo_s to make it flexible")
+
 def test_type_caching():
     ffi1 = FFI(); ffi1.cdef("struct foo_s;")
     ffi2 = FFI(); ffi2.cdef("struct foo_s;")    # different one!
@@ -260,8 +270,8 @@
     assert ffi.sizeof("struct foo_s") == 24  # found by the actual C code
     # lazily build the fields and boom:
     e = py.test.raises(ffi.error, ffi.new, "struct foo_s *")
-    assert str(e.value) == ("struct foo_s field 'a' was declared in the "
-                            "cdef to be 20 bytes, but is actually 24 bytes")
+    assert str(e.value).startswith("struct foo_s: wrong size for field 'a' "
+                                   "(cdef says 20, but C compiler says 24)")
 
 def test_open_array_in_struct():
     ffi = FFI()
diff --git a/new/test_verify1.py b/new/test_verify1.py
--- a/new/test_verify1.py
+++ b/new/test_verify1.py
@@ -402,41 +402,45 @@
     assert lib.bar(ffi.NULL) == 42
 
 def test_ffi_full_struct():
-    ffi = FFI()
-    ffi.cdef("struct foo_s { char x; int y; long *z; };")
-    ffi.verify("struct foo_s { char x; int y; long *z; };")
+    def check(verified_code):
+        ffi = FFI()
+        ffi.cdef("struct foo_s { char x; int y; long *z; };")
+        ffi.verify(verified_code)
+        ffi.new("struct foo_s *")
+
+    check("struct foo_s { char x; int y; long *z; };")
     #
     if sys.platform != 'win32':  # XXX fixme: only gives warnings
-        py.test.raises(VerificationError, ffi.verify,
+        py.test.raises(VerificationError, check,
             "struct foo_s { char x; int y; int *z; };")
     #
-    py.test.raises(VerificationError, ffi.verify,
-        "struct foo_s { int y; long *z; };")
+    py.test.raises(VerificationError, check,
+        "struct foo_s { int y; long *z; };")     # cdef'ed field x is missing
     #
-    e = py.test.raises(VerificationError, ffi.verify,
-        "struct foo_s { int y; char x; long *z; };")
-    assert str(e.value) == (
+    e = py.test.raises(FFI.error, check,
+                       "struct foo_s { int y; char x; long *z; };")
+    assert str(e.value).startswith(
         "struct foo_s: wrong offset for field 'x'"
-        " (we have 0, but C compiler says 4)")
+        " (cdef says 0, but C compiler says 4)")
     #
-    e = py.test.raises(VerificationError, ffi.verify,
+    e = py.test.raises(FFI.error, check,
         "struct foo_s { char x; int y; long *z; char extra; };")
-    assert str(e.value) == (
+    assert str(e.value).startswith(
         "struct foo_s: wrong total size"
-        " (we have %d, but C compiler says %d)" % (
-            ffi.sizeof("struct foo_s"),
-            ffi.sizeof("struct foo_s") + ffi.sizeof("long*")))
+        " (cdef says %d, but C compiler says %d)" % (
+            8 + FFI().sizeof('long *'),
+            8 + FFI().sizeof('long *') * 2))
     #
     # a corner case that we cannot really detect, but where it has no
     # bad consequences: the size is the same, but there is an extra field
     # that replaces what is just padding in our declaration above
-    ffi.verify("struct foo_s { char x, extra; int y; long *z; };")
+    check("struct foo_s { char x, extra; int y; long *z; };")
     #
-    e = py.test.raises(VerificationError, ffi.verify,
+    e = py.test.raises(FFI.error, check,
         "struct foo_s { char x; short pad; short y; long *z; };")
-    assert str(e.value) == (
+    assert str(e.value).startswith(
         "struct foo_s: wrong size for field 'y'"
-        " (we have 4, but C compiler says 2)")
+        " (cdef says 4, but C compiler says 2)")
 
 def test_ffi_nonfull_struct():
     ffi = FFI()

From noreply at buildbot.pypy.org  Sun Apr 19 10:45:27 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 19 Apr 2015 10:45:27 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: ffi.alignof()
Message-ID: <20150419084527.7C1E11C02AD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1771:1ae2580ab916
Date: 2015-04-19 10:40 +0200
http://bitbucket.org/cffi/cffi/changeset/1ae2580ab916/

Log:	ffi.alignof()

diff --git a/new/ffi_obj.c b/new/ffi_obj.c
--- a/new/ffi_obj.c
+++ b/new/ffi_obj.c
@@ -190,6 +190,23 @@
     return PyInt_FromSsize_t(ct->ct_size);
 }
 
+PyDoc_STRVAR(ffi_alignof_doc,
+"Return the natural alignment size in bytes of the argument.\n"
+"It can be a string naming a C type, or a 'cdata' instance.");
+
+static PyObject *ffi_alignof(FFIObject *self, PyObject *arg)
+{
+    int align;
+    CTypeDescrObject *ct = _ffi_type(self, arg, ACCEPT_ALL);
+    if (ct == NULL)
+        return NULL;
+
+    align = get_alignment(ct);
+    if (align < 0)
+        return NULL;
+    return PyInt_FromLong(align);
+}
+
 PyDoc_STRVAR(ffi_typeof_doc,
 "Parse the C type given as a string and return the\n"
 "corresponding  object.\n"
@@ -526,6 +543,7 @@
 #if 0
     {"addressof",     (PyCFunction)ffi_addressof, METH_VARARGS},
 #endif
+    {"alignof",       (PyCFunction)ffi_alignof,   METH_O,      ffi_alignof_doc},
     {"cast",          (PyCFunction)ffi_cast,      METH_VARARGS, ffi_cast_doc},
 #if 0
     {"close_library", ffi_close_library,          METH_VARARGS | METH_STATIC},
diff --git a/new/test_ffi_obj.py b/new/test_ffi_obj.py
--- a/new/test_ffi_obj.py
+++ b/new/test_ffi_obj.py
@@ -65,3 +65,10 @@
     ffi = _cffi1_backend.FFI()
     ffi.errno = 42
     assert ffi.errno == 42
+
+def test_ffi_alignof():
+    ffi = _cffi1_backend.FFI()
+    assert ffi.alignof("int") == 4
+    assert ffi.alignof("int[]") == 4
+    assert ffi.alignof("int[41]") == 4
+    assert ffi.alignof("short[41]") == 2

From noreply at buildbot.pypy.org  Sun Apr 19 10:45:28 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 19 Apr 2015 10:45:28 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: test and fix for ffi.sizeof()
Message-ID: <20150419084528.8B6851C02AD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1772:7a5750bbb806
Date: 2015-04-19 10:45 +0200
http://bitbucket.org/cffi/cffi/changeset/7a5750bbb806/

Log:	test and fix for ffi.sizeof()

diff --git a/new/ffi_obj.c b/new/ffi_obj.c
--- a/new/ffi_obj.c
+++ b/new/ffi_obj.c
@@ -178,16 +178,25 @@
 
 static PyObject *ffi_sizeof(FFIObject *self, PyObject *arg)
 {
+    Py_ssize_t size;
     CTypeDescrObject *ct = _ffi_type(self, arg, ACCEPT_ALL);
     if (ct == NULL)
         return NULL;
 
-    if (ct->ct_size < 0) {
+    size = ct->ct_size;
+
+    if (CData_Check(arg)) {
+        CDataObject *cd = (CDataObject *)arg;
+        if (cd->c_type->ct_flags & CT_ARRAY)
+            size = get_array_length(cd) * cd->c_type->ct_itemdescr->ct_size;
+    }
+
+    if (size < 0) {
         PyErr_Format(FFIError, "don't know the size of ctype '%s'",
                      ct->ct_name);
         return NULL;
     }
-    return PyInt_FromSsize_t(ct->ct_size);
+    return PyInt_FromSsize_t(size);
 }
 
 PyDoc_STRVAR(ffi_alignof_doc,
diff --git a/new/test_ffi_obj.py b/new/test_ffi_obj.py
--- a/new/test_ffi_obj.py
+++ b/new/test_ffi_obj.py
@@ -72,3 +72,13 @@
     assert ffi.alignof("int[]") == 4
     assert ffi.alignof("int[41]") == 4
     assert ffi.alignof("short[41]") == 2
+    assert ffi.alignof(ffi.new("int[41]")) == 4
+    assert ffi.alignof(ffi.new("int[]", 41)) == 4
+
+def test_ffi_sizeof():
+    ffi = _cffi1_backend.FFI()
+    assert ffi.sizeof("int") == 4
+    py.test.raises(ffi.error, ffi.sizeof, "int[]")
+    assert ffi.sizeof("int[41]") == 41 * 4
+    assert ffi.sizeof(ffi.new("int[41]")) == 41 * 4
+    assert ffi.sizeof(ffi.new("int[]", 41)) == 41 * 4

From noreply at buildbot.pypy.org  Sun Apr 19 14:07:48 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 19 Apr 2015 14:07:48 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof2: fix the test and hopefully fix the
	problem
Message-ID: <20150419120748.E45D31C0163@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof2
Changeset: r76830:2af4eb203367
Date: 2015-04-19 14:07 +0200
http://bitbucket.org/pypy/pypy/changeset/2af4eb203367/

Log:	fix the test and hopefully fix the problem

diff --git a/rpython/jit/backend/x86/test/test_recursive.py b/rpython/jit/backend/x86/test/test_recursive.py
--- a/rpython/jit/backend/x86/test/test_recursive.py
+++ b/rpython/jit/backend/x86/test/test_recursive.py
@@ -12,14 +12,19 @@
             return # this is 64 bit only check
 
         assert len(codemaps) == 3
+        # we want to create a map of differences, so unpacking the tracebacks
+        # byte by byte
         codemaps.sort(lambda a, b: cmp(a[1], b[1]))
         # biggest is the big loop, smallest is the bridge
-        assert unpack_traceback(codemaps[1][0]) == [2]
-        # XXX very specific ASM addresses, very fragile test, but what we can
-        #     do, really? 64bit only so far
-        assert unpack_traceback(codemaps[0][0]) == [2]
-        assert unpack_traceback(codemaps[1][0] + 100) == [2]
-        assert unpack_traceback(codemaps[2][0] + 100) == [4]
-        assert unpack_traceback(codemaps[2][0] + 200) == [4, 2]
-        assert unpack_traceback(codemaps[2][0] + 500) == [4, 2]
-        assert unpack_traceback(codemaps[2][0] + 380) == [4]
+        def get_ranges(c):
+            ranges = []
+            prev_traceback = None
+            for b in range(c[0], c[0] + c[1]):
+                tb = unpack_traceback(b)
+                if tb != prev_traceback:
+                    ranges.append(tb)
+                    prev_traceback = tb
+            return ranges
+        assert get_ranges(codemaps[2]) == [[4], [4, 2], [4], [4, 2], [4]]
+        assert get_ranges(codemaps[1]) == [[2]]
+        assert get_ranges(codemaps[0]) == [[2], []]
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1163,12 +1163,15 @@
             # itself, which has no result at all.
             assert len(self.metainterp.framestack) >= 2
             try:
-                self.metainterp.finishframe(None)
+                old_frame = self.metainterp.framestack[-1]
+                self.metainterp.finishframe(None, leave_portal_frame=False)
             except ChangeFrame:
                 pass
             frame = self.metainterp.framestack[-1]
             frame.do_recursive_call(jitdriver_sd, greenboxes + redboxes, orgpc,
                                     assembler_call=True)
+            jd_no = old_frame.jitcode.jitdriver_sd.index
+            self.metainterp.leave_portal_frame(jd_no)
             raise ChangeFrame
 
     def debug_merge_point(self, jitdriver_sd, jd_index, portal_call_depth, current_call_id, greenkey):
@@ -1808,12 +1811,13 @@
         self.history.record(rop.LEAVE_PORTAL_FRAME, [ConstInt(jd_no)], None)
 
 
-    def popframe(self):
+    def popframe(self, leave_portal_frame=True):
         frame = self.framestack.pop()
         jitcode = frame.jitcode
         if jitcode.jitdriver_sd:
             self.portal_call_depth -= 1
-            self.leave_portal_frame(jitcode.jitdriver_sd.index)
+            if leave_portal_frame:
+                self.leave_portal_frame(jitcode.jitdriver_sd.index)
             self.call_ids.pop()
         if frame.greenkey is not None and self.is_main_jitcode(jitcode):
             self.portal_trace_positions.append(
@@ -1824,10 +1828,10 @@
         frame.cleanup_registers()
         self.free_frames_list.append(frame)
 
-    def finishframe(self, resultbox):
+    def finishframe(self, resultbox, leave_portal_frame=True):
         # handle a non-exceptional return from the current frame
         self.last_exc_value_box = None
-        self.popframe()
+        self.popframe(leave_portal_frame=leave_portal_frame)
         if self.framestack:
             if resultbox is not None:
                 self.framestack[-1].make_result_of_lastop(resultbox)

From noreply at buildbot.pypy.org  Sun Apr 19 14:09:30 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 19 Apr 2015 14:09:30 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof2: fix rpython
Message-ID: <20150419120930.A02D71C0163@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof2
Changeset: r76831:3b3feb9827e1
Date: 2015-04-19 14:09 +0200
http://bitbucket.org/pypy/pypy/changeset/3b3feb9827e1/

Log:	fix rpython

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1162,8 +1162,8 @@
             # it must be the call to 'self', and not the jit_merge_point
             # itself, which has no result at all.
             assert len(self.metainterp.framestack) >= 2
+            old_frame = self.metainterp.framestack[-1]
             try:
-                old_frame = self.metainterp.framestack[-1]
                 self.metainterp.finishframe(None, leave_portal_frame=False)
             except ChangeFrame:
                 pass

From noreply at buildbot.pypy.org  Sun Apr 19 15:02:47 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 19 Apr 2015 15:02:47 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof2: use -1 as a special value
Message-ID: <20150419130247.A3FFC1C137A@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof2
Changeset: r76832:680ec700b161
Date: 2015-04-19 15:02 +0200
http://bitbucket.org/pypy/pypy/changeset/680ec700b161/

Log:	use -1 as a special value

diff --git a/pypy/module/_vmprof/src/get_custom_offset.c b/pypy/module/_vmprof/src/get_custom_offset.c
--- a/pypy/module/_vmprof/src/get_custom_offset.c
+++ b/pypy/module/_vmprof/src/get_custom_offset.c
@@ -43,7 +43,7 @@
     start = n;
     while (n < max_depth) {
         id = pypy_yield_codemap_at_addr(codemap, addr, ¤t_pos);
-        if (id == 0)
+        if (id == -1)
             // finish
             break;
         result[n++] = (void *)id;
diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py
--- a/rpython/jit/backend/llsupport/codemap.py
+++ b/rpython/jit/backend/llsupport/codemap.py
@@ -132,7 +132,7 @@
     res = []
     while True:
         item = yield_bytecode_at_addr(codemap_raw, addr, storage)
-        if item == 0:
+        if item == -1:
             break
         res.append(item)
     lltype.free(storage, flavor='raw')
diff --git a/rpython/jit/backend/llsupport/src/codemap.c b/rpython/jit/backend/llsupport/src/codemap.c
--- a/rpython/jit/backend/llsupport/src/codemap.c
+++ b/rpython/jit/backend/llsupport/src/codemap.c
@@ -115,10 +115,10 @@
 
     while (1) {
         if (current_pos >= data->bytecode_info_size)
-            return 0;
+            return -1;
         next_start = data->bytecode_info[current_pos + 1];
         if (next_start > rel_addr)
-            return 0;
+            return -1;
         next_stop = data->bytecode_info[current_pos + 2];
         if (next_stop > rel_addr) {
             *current_pos_addr = current_pos + 4;

From noreply at buildbot.pypy.org  Sun Apr 19 17:06:02 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 19 Apr 2015 17:06:02 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof2: make lldebug0 increase the stack size
Message-ID: <20150419150602.7EDA11C13FD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof2
Changeset: r76833:b5a9393ab3ea
Date: 2015-04-19 17:06 +0200
http://bitbucket.org/pypy/pypy/changeset/b5a9393ab3ea/

Log:	make lldebug0 increase the stack size

diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -407,7 +407,7 @@
             ('debug_mem', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DPYPY_USE_TRIVIAL_MALLOC" debug_target'),
             ('llsafer', '', '$(MAKE) CFLAGS="-O2 -DRPY_LL_ASSERT" $(DEFAULT_TARGET)'),
             ('lldebug', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DRPY_LL_ASSERT" debug_target'),
-            ('lldebug0','', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -O0 -DRPY_ASSERT -DRPY_LL_ASSERT" debug_target'),
+            ('lldebug0','', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -O0 -DMAX_STACK_SIZE=8192000 -DRPY_ASSERT -DRPY_LL_ASSERT" debug_target'),
             ('profile', '', '$(MAKE) CFLAGS="-g -O1 -pg $(CFLAGS) -fno-omit-frame-pointer" LDFLAGS="-pg $(LDFLAGS)" $(DEFAULT_TARGET)'),
             ]
         if self.has_profopt():

From noreply at buildbot.pypy.org  Sun Apr 19 17:50:43 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 19 Apr 2015 17:50:43 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof2: try yet a different approach - enclose
 stuff in 0x2 0x3 instead of stripping (the stripping has a problem of not
 working when we have only one
Message-ID: <20150419155043.8B6E01C0163@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof2
Changeset: r76834:8423127b5892
Date: 2015-04-19 17:50 +0200
http://bitbucket.org/pypy/pypy/changeset/8423127b5892/

Log:	try yet a different approach - enclose stuff in 0x2 0x3 instead of
	stripping (the stripping has a problem of not working when we have
	only one

diff --git a/pypy/module/_vmprof/src/get_custom_offset.c b/pypy/module/_vmprof/src/get_custom_offset.c
--- a/pypy/module/_vmprof/src/get_custom_offset.c
+++ b/pypy/module/_vmprof/src/get_custom_offset.c
@@ -40,17 +40,18 @@
     // modify the last entry to point to start address and not the random one
     // in the middle
     result[n - 1] = (void*)start_addr;
+    result[n] = (void*)2;
+    n++;
     start = n;
     while (n < max_depth) {
         id = pypy_yield_codemap_at_addr(codemap, addr, ¤t_pos);
         if (id == -1)
             // finish
             break;
+        if (id == 0)
+            continue; // not main codemap
         result[n++] = (void *)id;
     }
-    // we strip the topmost part - the reason is that it's either
-    // represented in the jitted caller or it's not jitted (we have the
-    // same function essentially twice
     k = 0;
     while (k < (n - start) / 2) {
         tmp = result[start + k];
@@ -58,8 +59,8 @@
         result[n - k - 1] = tmp;
         k++;
     }
-    if (n != max_depth) {
-        n--;
+    if (n < max_depth) {
+        result[n++] = (void*)3;
     }
     return n;
 }
diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py
--- a/rpython/jit/backend/llsupport/codemap.py
+++ b/rpython/jit/backend/llsupport/codemap.py
@@ -146,8 +146,6 @@
         self.last_call_depth = -1
 
     def enter_portal_frame(self, jd_id, unique_id, relpos):
-        if jd_id != 0:
-            return
         self.l.append(unique_id)
         self.l.append(relpos)
         self.patch_position.append(len(self.l))
@@ -155,8 +153,6 @@
         self.l.append(0) # second marker
 
     def leave_portal_frame(self, jd_id, relpos):
-        if jd_id != 0:
-            return
         to_patch = self.patch_position.pop()
         self.l[to_patch] = relpos
         self.l[to_patch + 1] = len(self.l)

From noreply at buildbot.pypy.org  Sun Apr 19 19:33:37 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 19 Apr 2015 19:33:37 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof2: due to the last checkin we no longer
	need to do this
Message-ID: <20150419173337.3AA771C1377@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof2
Changeset: r76835:c279ae5451ba
Date: 2015-04-19 19:33 +0200
http://bitbucket.org/pypy/pypy/changeset/c279ae5451ba/

Log:	due to the last checkin we no longer need to do this

diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -97,8 +97,7 @@
             gc_frame = cast_instance_to_gcref(frame)
             gc_inputvalue = cast_instance_to_gcref(w_inputvalue)
             gc_operr = cast_instance_to_gcref(operr)
-            assert frame.pycode._unique_id & 3 == 0
-            unique_id = frame.pycode._unique_id | 1
+            unique_id = frame.pycode._unique_id
             gc_result = pypy_execute_frame_trampoline(gc_frame, gc_inputvalue,
                                                       gc_operr, unique_id)
             return cast_base_ptr_to_instance(W_Root, gc_result)

From noreply at buildbot.pypy.org  Sun Apr 19 22:31:45 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 19 Apr 2015 22:31:45 +0200 (CEST)
Subject: [pypy-commit] pypy object-dtype2: add test, fix bool
Message-ID: <20150419203145.16CC71C137A@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: object-dtype2
Changeset: r76836:f86e55aeb94a
Date: 2015-04-19 18:48 +0300
http://bitbucket.org/pypy/pypy/changeset/f86e55aeb94a/

Log:	add test, fix bool

diff --git a/pypy/module/micronumpy/test/test_object_arrays.py b/pypy/module/micronumpy/test/test_object_arrays.py
--- a/pypy/module/micronumpy/test/test_object_arrays.py
+++ b/pypy/module/micronumpy/test/test_object_arrays.py
@@ -35,6 +35,8 @@
         assert b == complex(1, -1)
         b = np.array(complex(1, -1)) and a
         assert (b == a).all()
+        c = np.array([1, 2, 3])
+        assert (a[0] != c[0])
 
     def test_logical_ufunc(self):
         import numpy as np
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1746,6 +1746,7 @@
             return v1
         return v2
 
+    @raw_unary_op
     def bool(self,v):
         return self._obool(v)
 

From noreply at buildbot.pypy.org  Sun Apr 19 22:31:46 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 19 Apr 2015 22:31:46 +0200 (CEST)
Subject: [pypy-commit] pypy object-dtype2: prefer obscure exception to a
	segfault
Message-ID: <20150419203146.4419E1C137A@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: object-dtype2
Changeset: r76837:df509dc69d3e
Date: 2015-04-19 23:25 +0300
http://bitbucket.org/pypy/pypy/changeset/df509dc69d3e/

Log:	prefer obscure exception to a segfault

diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1686,9 +1686,12 @@
             self._write(storage, i, offset, value, gcstruct)
 
     def unbox(self, box):
-        assert isinstance(box, self.BoxType)
-        return box.w_obj
-
+        if isinstance(box, self.BoxType):
+            return box.w_obj
+        else:
+            raise oefmt(self.space.w_NotImplementedError,
+                "object dtype cannot unbox %s", str(box))
+            
     @specialize.argtype(1)
     def box(self, w_obj):
         if isinstance(w_obj, W_Root):

From noreply at buildbot.pypy.org  Sun Apr 19 22:31:47 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 19 Apr 2015 22:31:47 +0200 (CEST)
Subject: [pypy-commit] pypy object-dtype2: start to add creation of array
 from __array_interface__, should finish this on another branch
Message-ID: <20150419203147.63B1A1C137A@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: object-dtype2
Changeset: r76838:f8e96786ec62
Date: 2015-04-19 23:26 +0300
http://bitbucket.org/pypy/pypy/changeset/f8e96786ec62/

Log:	start to add creation of array from __array_interface__, should
	finish this on another branch

diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -38,6 +38,31 @@
         raise oefmt(space.w_ValueError,
                     "object __array__ method not producing an array")
 
+def try_interface_method(space, w_object):
+    try:
+        w_interface = space.getattr(w_object, space.wrap("__array_interface__"))
+    except OperationError, e:
+        if e.match(space, space.w_AttributeError):
+            return None
+        raise
+    version = space.int_w(space.finditem(w_interface, space.wrap("version")))
+    if version < 3:
+        raise oefmt(space.w_NotImplementedError,
+                "__array_interface__ version %d not supported", version)
+    # make a view into the data
+    w_shape = space.finditem(w_interface, space.wrap('shape'))
+    w_dtype = space.finditem(w_interface, space.wrap('typestr'))
+    w_descr = space.finditem(w_interface, space.wrap('descr'))
+    data_w = space.listview(space.finditem(w_interface, space.wrap('data')))
+    w_strides = space.finditem(w_interface, space.wrap('strides'))
+    shape = [space.int_w(i) for i in space.listview(w_shape)]
+    dtype = descriptor.decode_w_dtype(space, w_dtype)
+    rw = space.is_true(data_w[1])
+    #print 'create view from shape',shape,'dtype',dtype,'descr',w_descr,'data',data_w[0],'rw',rw
+    raise oefmt(space.w_NotImplementedError,
+                "creating array from __array_interface__ not supported yet")
+    return 
+    
 
 @unwrap_spec(ndmin=int, copy=bool, subok=bool)
 def array(space, w_object, w_dtype=None, copy=True, w_order=None, subok=False,
@@ -63,7 +88,11 @@
             # continue with w_array, but do further operations in place
             w_object = w_array
             copy = False
-
+    if not isinstance(w_object, W_NDimArray):
+        w_array = try_interface_method(space, w_object)
+        if w_array is not None:
+            w_object = w_array
+            copy = False
     dtype = descriptor.decode_w_dtype(space, w_dtype)
 
     if space.is_none(w_order):
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -551,6 +551,7 @@
             else:
                 strides = self.descr_get_strides(space)
             space.setitem_str(w_d, 'strides', strides)
+            space.setitem_str(w_d, 'version', space.wrap(3))
             return w_d
 
     w_pypy_data = None
diff --git a/pypy/module/micronumpy/test/test_object_arrays.py b/pypy/module/micronumpy/test/test_object_arrays.py
--- a/pypy/module/micronumpy/test/test_object_arrays.py
+++ b/pypy/module/micronumpy/test/test_object_arrays.py
@@ -107,4 +107,6 @@
         interface['shape'] = tuple([3])
         interface['strides'] = tuple([0])
         c = np.array(DummyArray(interface, base=a))
+        c.dtype = a.dtype
+        #print c
         assert (c == np.array([(1, 2, 3), (1, 2, 3), (1, 2, 3)], dtype='u4,u4,u4') ).all()

From noreply at buildbot.pypy.org  Mon Apr 20 00:35:54 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Mon, 20 Apr 2015 00:35:54 +0200 (CEST)
Subject: [pypy-commit] pypy default: Issue2033: A descriptor with __delete__
 and no __get__ is also a data descriptor.
Message-ID: <20150419223554.5269B1C1D8F@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: 
Changeset: r76839:e822667de276
Date: 2015-04-20 00:35 +0200
http://bitbucket.org/pypy/pypy/changeset/e822667de276/

Log:	Issue2033: A descriptor with __delete__ and no __get__ is also a
	data descriptor.

	(CPython has only slot for __get__ and __delete__: tp_descr_set)

diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -129,7 +129,8 @@
     # This is meant to be a *mixin*.
 
     def is_data_descr(space, w_obj):
-        return space.lookup(w_obj, '__set__') is not None
+        return (space.lookup(w_obj, '__set__') is not None or
+                space.lookup(w_obj, '__delete__') is not None)
 
     def get_and_call_args(space, w_descr, w_obj, args):
         # a special case for performance and to avoid infinite recursion
diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py
--- a/pypy/objspace/test/test_descroperation.py
+++ b/pypy/objspace/test/test_descroperation.py
@@ -607,6 +607,18 @@
 
         raises(AttributeError, lambda: A().a)
 
+    def test_delete_descriptor(self):
+        class Prop(object):
+            def __get__(self, obj, cls):
+                return 42
+            def __delete__(self, obj):
+                obj.deleted = True
+        class C(object):
+            x = Prop()
+        obj = C()
+        del obj.x
+        assert obj.deleted
+
     def test_non_callable(self):
         meth = classmethod(1).__get__(1)
         raises(TypeError, meth)

From noreply at buildbot.pypy.org  Mon Apr 20 04:52:39 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 20 Apr 2015 04:52:39 +0200 (CEST)
Subject: [pypy-commit] pypy callfamily: fix missing import
Message-ID: <20150420025239.5B1D41C0163@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: callfamily
Changeset: r76840:745aaafc86f8
Date: 2015-04-19 15:15 +0100
http://bitbucket.org/pypy/pypy/changeset/745aaafc86f8/

Log:	fix missing import

diff --git a/rpython/rtyper/rpbc.py b/rpython/rtyper/rpbc.py
--- a/rpython/rtyper/rpbc.py
+++ b/rpython/rtyper/rpbc.py
@@ -1,5 +1,6 @@
 import types
 
+from rpython.flowspace.model import FunctionGraph, Link, Block, SpaceOperation
 from rpython.annotator import model as annmodel
 from rpython.annotator.description import (
     FunctionDesc, ClassDesc, MethodDesc, FrozenDesc, MethodOfFrozenDesc)

From noreply at buildbot.pypy.org  Mon Apr 20 04:52:40 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 20 Apr 2015 04:52:40 +0200 (CEST)
Subject: [pypy-commit] pypy callfamily: cleanup
Message-ID: <20150420025240.96CF31C0163@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: callfamily
Changeset: r76841:053f28ada18a
Date: 2015-04-20 03:52 +0100
http://bitbucket.org/pypy/pypy/changeset/053f28ada18a/

Log:	cleanup

diff --git a/rpython/rtyper/rpbc.py b/rpython/rtyper/rpbc.py
--- a/rpython/rtyper/rpbc.py
+++ b/rpython/rtyper/rpbc.py
@@ -70,6 +70,17 @@
 
 class ConcreteCallTableRow(dict):
     """A row in a concrete call table."""
+    @classmethod
+    def from_row(cls, rtyper, row):
+        concreterow = cls()
+        for funcdesc, graph in row.items():
+            llfn = rtyper.getcallable(graph)
+            concreterow[funcdesc] = llfn
+        assert len(concreterow) > 0
+        # 'typeOf(llfn)' should be the same for all graphs
+        concreterow.fntype = typeOf(llfn)
+        return concreterow
+
 
 class LLCallTable(object):
     """A call table of a call family with low-level functions."""
@@ -124,17 +135,9 @@
     concreterows = {}
     for shape, rows in callfamily.calltables.items():
         for index, row in enumerate(rows):
-            concreterow = ConcreteCallTableRow()
-            for funcdesc, graph in row.items():
-                llfn = rtyper.getcallable(graph)
-                concreterow[funcdesc] = llfn
-            assert len(concreterow) > 0
-            # 'llfn' should be the same for all graphs
-            concreterow.fntype = typeOf(llfn)
+            concreterow = ConcreteCallTableRow.from_row(rtyper, row)
             concreterows[shape, index] = concreterow
-
-    for row in concreterows.values():
-        llct.add(row)
+            llct.add(concreterow)
 
     for (shape, index), row in concreterows.items():
         existingindex, biggerrow = llct.lookup(row)

From noreply at buildbot.pypy.org  Mon Apr 20 11:01:24 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 20 Apr 2015 11:01:24 +0200 (CEST)
Subject: [pypy-commit] pypy default: Print a traceback when we get an
 exception (bogus gdb interface, why doesn't it do it)
Message-ID: <20150420090124.A267E1C0A29@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76843:b0f133375831
Date: 2015-04-20 11:00 +0200
http://bitbucket.org/pypy/pypy/changeset/b0f133375831/

Log:	Print a traceback when we get an exception (bogus gdb interface, why
	doesn't it do it)

diff --git a/pypy/tool/gdb_pypy.py b/pypy/tool/gdb_pypy.py
--- a/pypy/tool/gdb_pypy.py
+++ b/pypy/tool/gdb_pypy.py
@@ -73,15 +73,20 @@
 
     def invoke(self, arg, from_tty):
         # some magic code to automatically reload the python file while developing
-        from pypy.tool import gdb_pypy
         try:
-            reload(gdb_pypy)
+            from pypy.tool import gdb_pypy
+            try:
+                reload(gdb_pypy)
+            except:
+                import imp
+                imp.reload(gdb_pypy)
+            gdb_pypy.RPyType.prog2typeids = self.prog2typeids # persist the cache
+            self.__class__ = gdb_pypy.RPyType
+            print (self.do_invoke(arg, from_tty).decode('latin-1'))
         except:
-            import imp
-            imp.reload(gdb_pypy)
-        gdb_pypy.RPyType.prog2typeids = self.prog2typeids # persist the cache
-        self.__class__ = gdb_pypy.RPyType
-        print (self.do_invoke(arg, from_tty).decode('latin-1'))
+            import traceback
+            traceback.print_exc()
+            raise
 
     def do_invoke(self, arg, from_tty):
         try:

From noreply at buildbot.pypy.org  Mon Apr 20 11:01:25 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 20 Apr 2015 11:01:25 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix for Py3 support
Message-ID: <20150420090125.BD7E41C0A29@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76844:9f9e646594bd
Date: 2015-04-20 11:01 +0200
http://bitbucket.org/pypy/pypy/changeset/9f9e646594bd/

Log:	Fix for Py3 support

diff --git a/pypy/tool/gdb_pypy.py b/pypy/tool/gdb_pypy.py
--- a/pypy/tool/gdb_pypy.py
+++ b/pypy/tool/gdb_pypy.py
@@ -134,7 +134,7 @@
         try:
             self.gdb.execute('dump binary memory %s %s %s+%d' %
                              (fname, vstart, vstart, length))
-            with open(fname, 'rt') as fobj:
+            with open(fname, 'rb') as fobj:
                 data = fobj.read()
             return TypeIdsMap(zlib.decompress(data).splitlines(True), self.gdb)
         finally:

From noreply at buildbot.pypy.org  Mon Apr 20 11:01:23 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 20 Apr 2015 11:01:23 +0200 (CEST)
Subject: [pypy-commit] pypy default: Recognizes when we get a pointer
 expression, and dereference it automatically
Message-ID: <20150420090123.75D311C0A29@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76842:1b705dee117d
Date: 2015-04-20 10:58 +0200
http://bitbucket.org/pypy/pypy/changeset/1b705dee117d/

Log:	Recognizes when we get a pointer expression, and dereference it
	automatically

diff --git a/pypy/tool/gdb_pypy.py b/pypy/tool/gdb_pypy.py
--- a/pypy/tool/gdb_pypy.py
+++ b/pypy/tool/gdb_pypy.py
@@ -55,10 +55,10 @@
 
 class RPyType(Command):
     """
-    Prints the RPython type of the expression (remember to dereference it!)
+    Prints the RPython type of the expression.
     E.g.:
 
-    (gdb) rpy_type *l_v123
+    (gdb) rpy_type l_v123
     GcStruct pypy.foo.Bar { super, inst_xxx, inst_yyy }
     """
 
@@ -88,6 +88,8 @@
             offset = int(arg)
         except ValueError:
             obj = self.gdb.parse_and_eval(arg)
+            if obj.type.code == self.gdb.TYPE_CODE_PTR:
+                obj = obj.dereference()
             hdr = lookup(obj, '_gcheader')
             tid = hdr['h_tid']
             if sys.maxsize < 2**32:

From noreply at buildbot.pypy.org  Mon Apr 20 11:10:46 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 20 Apr 2015 11:10:46 +0200 (CEST)
Subject: [pypy-commit] pypy default: more py3 fixes
Message-ID: <20150420091046.871331C0163@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76845:8f152d9a1096
Date: 2015-04-20 11:10 +0200
http://bitbucket.org/pypy/pypy/changeset/8f152d9a1096/

Log:	more py3 fixes

diff --git a/pypy/tool/gdb_pypy.py b/pypy/tool/gdb_pypy.py
--- a/pypy/tool/gdb_pypy.py
+++ b/pypy/tool/gdb_pypy.py
@@ -82,11 +82,13 @@
                 imp.reload(gdb_pypy)
             gdb_pypy.RPyType.prog2typeids = self.prog2typeids # persist the cache
             self.__class__ = gdb_pypy.RPyType
-            print (self.do_invoke(arg, from_tty).decode('latin-1'))
+            result = self.do_invoke(arg, from_tty)
+            if not isinstance(result, str):
+                result = result.decode('latin-1')
+            print(result)
         except:
             import traceback
             traceback.print_exc()
-            raise
 
     def do_invoke(self, arg, from_tty):
         try:
@@ -107,7 +109,7 @@
         if offset in typeids:
             return typeids[offset]
         else:
-            return 'Cannot find the type with offset %d' % offset
+            return 'Cannot find the type with offset 0x%x' % offset
 
     def get_typeids(self):
         try:

From noreply at buildbot.pypy.org  Mon Apr 20 11:12:11 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 20 Apr 2015 11:12:11 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Fix for gdb not printing the traceback
Message-ID: <20150420091211.D80E91C0163@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r1756:12b2b35a4fb5
Date: 2015-04-20 11:12 +0200
http://bitbucket.org/pypy/stmgc/changeset/12b2b35a4fb5/

Log:	Fix for gdb not printing the traceback

diff --git a/c7/gdb/gdb_stm.py b/c7/gdb/gdb_stm.py
--- a/c7/gdb/gdb_stm.py
+++ b/c7/gdb/gdb_stm.py
@@ -25,7 +25,12 @@
 def gdb_function(func):
     class Func(gdb.Function):
         __doc__ = func.__doc__
-        invoke = staticmethod(func)
+        def invoke(self, *args, **kwds):
+            try:
+                func(*args, **kwds)
+            except:
+                import traceback
+                traceback.print_exc()
     Func(func.__name__)
 
 # -------------------------------------------------------

From noreply at buildbot.pypy.org  Mon Apr 20 11:28:10 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 20 Apr 2015 11:28:10 +0200 (CEST)
Subject: [pypy-commit] pypy default: Pretty-print arrays in addition to lists
Message-ID: <20150420092810.40D2C1C0A29@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76846:19c8e7f2ca21
Date: 2015-04-20 11:28 +0200
http://bitbucket.org/pypy/pypy/changeset/19c8e7f2ca21/

Log:	Pretty-print arrays in addition to lists

diff --git a/pypy/tool/gdb_pypy.py b/pypy/tool/gdb_pypy.py
--- a/pypy/tool/gdb_pypy.py
+++ b/pypy/tool/gdb_pypy.py
@@ -258,29 +258,37 @@
     def lookup(cls, val, gdb=None):
         t = val.type
         if (is_ptr(t, gdb) and t.target().tag is not None and
-            re.match(r'pypy_list\d*', t.target().tag)):
+            re.match(r'pypy_(list|array)\d*', t.target().tag)):
             return cls(val)
         return None
 
     def to_string(self):
-        length = int(self.val['l_length'])
-        array = self.val['l_items']
-        allocated = int(array['length'])
-        items = array['items']
+        t = self.val.type
+        print(t)
+        if t.target().tag.startswith(r'pypy_array'):
+            length = int(self.val['length'])
+            items = self.val['items']
+            allocstr = ''
+        else:
+            length = int(self.val['l_length'])
+            array = self.val['l_items']
+            allocated = int(array['length'])
+            items = array['items']
+            allocstr = ', alloc=%d' % allocated
         itemlist = []
         for i in range(length):
             item = items[i]
             itemlist.append(str(item))
         str_items = ', '.join(itemlist)
-        return 'r[%s] (len=%d, alloc=%d)' % (str_items, length, allocated)
+        return 'r[%s] (len=%d%s)' % (str_items, length, allocstr)
 
 
 try:
     import gdb
     RPyType() # side effects
-    gdb.pretty_printers += [
+    gdb.pretty_printers = [
         RPyStringPrinter.lookup,
         RPyListPrinter.lookup
-        ]
+        ] + gdb.pretty_printers
 except ImportError:
     pass

From noreply at buildbot.pypy.org  Mon Apr 20 11:37:40 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 20 Apr 2015 11:37:40 +0200 (CEST)
Subject: [pypy-commit] stmgc default: oups, inspired by a Command in pypy,
 but this is actually a Function
Message-ID: <20150420093740.6F0C51C0FCB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r1757:c98fd943557c
Date: 2015-04-20 11:38 +0200
http://bitbucket.org/pypy/stmgc/changeset/c98fd943557c/

Log:	oups, inspired by a Command in pypy, but this is actually a Function

diff --git a/c7/gdb/gdb_stm.py b/c7/gdb/gdb_stm.py
--- a/c7/gdb/gdb_stm.py
+++ b/c7/gdb/gdb_stm.py
@@ -27,10 +27,11 @@
         __doc__ = func.__doc__
         def invoke(self, *args, **kwds):
             try:
-                func(*args, **kwds)
+                return func(*args, **kwds)
             except:
                 import traceback
                 traceback.print_exc()
+                raise
     Func(func.__name__)
 
 # -------------------------------------------------------

From noreply at buildbot.pypy.org  Mon Apr 20 13:49:47 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 20 Apr 2015 13:49:47 +0200 (CEST)
Subject: [pypy-commit] pypy default: a python3-friendly command is needed
Message-ID: <20150420114947.4DE391C0163@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76847:2066bbb2c82a
Date: 2015-04-20 13:29 +0200
http://bitbucket.org/pypy/pypy/changeset/2066bbb2c82a/

Log:	a python3-friendly command is needed

diff --git a/pypy/tool/gdb_pypy.py b/pypy/tool/gdb_pypy.py
--- a/pypy/tool/gdb_pypy.py
+++ b/pypy/tool/gdb_pypy.py
@@ -5,7 +5,7 @@
 
 Or, alternatively:
 
-(gdb) python execfile('/path/to/gdb_pypy.py')
+(gdb) python exec(open('/path/to/gdb_pypy.py').read())
 """
 
 import re

From noreply at buildbot.pypy.org  Mon Apr 20 13:49:48 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 20 Apr 2015 13:49:48 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add a test for pprinting arrays
Message-ID: <20150420114948.6FD4F1C0163@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76848:0e1434844851
Date: 2015-04-20 13:31 +0200
http://bitbucket.org/pypy/pypy/changeset/0e1434844851/

Log:	Add a test for pprinting arrays

diff --git a/pypy/tool/test/test_gdb_pypy.py b/pypy/tool/test/test_gdb_pypy.py
--- a/pypy/tool/test/test_gdb_pypy.py
+++ b/pypy/tool/test/test_gdb_pypy.py
@@ -204,6 +204,12 @@
     mylist.type.target().tag = None
     assert gdb_pypy.RPyListPrinter.lookup(mylist, FakeGdb) is None
 
+def test_pprint_array():
+    d = {'_gcheder': {'h_tid': 234}, 'length': 3, 'items': [20, 21, 22]}
+    mylist = PtrValue(d, type_tag='pypy_array1')
+    printer = gdb_pypy.RPyListPrinter.lookup(mylist, FakeGdb)
+    assert printer.to_string() == 'r[20, 21, 22] (len=3)'
+
 def test_typeidsmap():
     gdb = FakeGdb('', {exprmember(1): 111,
                        exprmember(2): 222,

From noreply at buildbot.pypy.org  Mon Apr 20 13:49:49 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 20 Apr 2015 13:49:49 +0200 (CEST)
Subject: [pypy-commit] pypy default: Null arrays and lists
Message-ID: <20150420114949.876C11C0163@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76849:55bd99ad0ea8
Date: 2015-04-20 13:40 +0200
http://bitbucket.org/pypy/pypy/changeset/55bd99ad0ea8/

Log:	Null arrays and lists

diff --git a/pypy/tool/gdb_pypy.py b/pypy/tool/gdb_pypy.py
--- a/pypy/tool/gdb_pypy.py
+++ b/pypy/tool/gdb_pypy.py
@@ -264,12 +264,15 @@
 
     def to_string(self):
         t = self.val.type
-        print(t)
         if t.target().tag.startswith(r'pypy_array'):
+            if not self.val:
+                return 'r(null_array)'
             length = int(self.val['length'])
             items = self.val['items']
             allocstr = ''
         else:
+            if not self.val:
+                return 'r(null_list)'
             length = int(self.val['l_length'])
             array = self.val['l_items']
             allocated = int(array['length'])
diff --git a/pypy/tool/test/test_gdb_pypy.py b/pypy/tool/test/test_gdb_pypy.py
--- a/pypy/tool/test/test_gdb_pypy.py
+++ b/pypy/tool/test/test_gdb_pypy.py
@@ -210,6 +210,16 @@
     printer = gdb_pypy.RPyListPrinter.lookup(mylist, FakeGdb)
     assert printer.to_string() == 'r[20, 21, 22] (len=3)'
 
+def test_pprint_null_list():
+    mylist = PtrValue({}, type_tag='pypy_list1')
+    printer = gdb_pypy.RPyListPrinter.lookup(mylist, FakeGdb)
+    assert printer.to_string() == 'r(null_list)'
+
+def test_pprint_null_array():
+    mylist = PtrValue({}, type_tag='pypy_array1')
+    printer = gdb_pypy.RPyListPrinter.lookup(mylist, FakeGdb)
+    assert printer.to_string() == 'r(null_array)'
+
 def test_typeidsmap():
     gdb = FakeGdb('', {exprmember(1): 111,
                        exprmember(2): 222,

From noreply at buildbot.pypy.org  Mon Apr 20 13:49:50 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 20 Apr 2015 13:49:50 +0200 (CEST)
Subject: [pypy-commit] pypy default: Try to protect against recursion
Message-ID: <20150420114950.A3F8C1C0163@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76850:d1780357bcfa
Date: 2015-04-20 13:49 +0200
http://bitbucket.org/pypy/pypy/changeset/d1780357bcfa/

Log:	Try to protect against recursion

diff --git a/pypy/tool/gdb_pypy.py b/pypy/tool/gdb_pypy.py
--- a/pypy/tool/gdb_pypy.py
+++ b/pypy/tool/gdb_pypy.py
@@ -251,6 +251,8 @@
     fields
     """
 
+    recursive = False
+
     def __init__(self, val):
         self.val = val
 
@@ -278,11 +280,18 @@
             allocated = int(array['length'])
             items = array['items']
             allocstr = ', alloc=%d' % allocated
-        itemlist = []
-        for i in range(length):
-            item = items[i]
-            itemlist.append(str(item))
-        str_items = ', '.join(itemlist)
+        if RPyListPrinter.recursive:
+            str_items = '...'
+        else:
+            RPyListPrinter.recursive = True
+            try:
+                itemlist = []
+                for i in range(length):
+                    item = items[i]
+                    itemlist.append(str(item))    # may recurse here
+                str_items = ', '.join(itemlist)
+            finally:
+                RPyListPrinter.recursive = False
         return 'r[%s] (len=%d%s)' % (str_items, length, allocstr)
 
 

From noreply at buildbot.pypy.org  Mon Apr 20 17:32:28 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 20 Apr 2015 17:32:28 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof2: Test and fix: due to alignment,
 the address given to pypy_jit_codemap_del()
Message-ID: <20150420153228.22FCE1C08AF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: vmprof2
Changeset: r76852:02cfea4f1675
Date: 2015-04-20 17:32 +0200
http://bitbucket.org/pypy/pypy/changeset/02cfea4f1675/

Log:	Test and fix: due to alignment, the address given to
	pypy_jit_codemap_del() might be slightly lower than the address
	given to pypy_jit_codemap_add().

diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py
--- a/rpython/jit/backend/llsupport/codemap.py
+++ b/rpython/jit/backend/llsupport/codemap.py
@@ -31,7 +31,7 @@
                                      unsigned int machine_code_size,
                                      long *bytecode_info,
                                      unsigned int bytecode_info_size);
-RPY_EXTERN long *pypy_jit_codemap_del(uintptr_t addr);
+RPY_EXTERN long *pypy_jit_codemap_del(uintptr_t addr, unsigned int size);
 RPY_EXTERN uintptr_t pypy_jit_codemap_firstkey(void);
 RPY_EXTERN void *pypy_find_codemap_at_addr(long addr, long* start_addr);
 RPY_EXTERN long pypy_yield_codemap_at_addr(void *codemap_raw, long addr,
@@ -55,7 +55,7 @@
                                    INT_LIST_PTR, lltype.Signed],
                                   lltype.Signed)
 pypy_jit_codemap_del = llexternal('pypy_jit_codemap_del',
-                                  [lltype.Signed], INT_LIST_PTR)
+                                  [lltype.Signed, lltype.Signed], INT_LIST_PTR)
 pypy_jit_codemap_firstkey = llexternal('pypy_jit_codemap_firstkey',
                                        [], lltype.Signed)
 
@@ -89,11 +89,11 @@
             key = pypy_jit_codemap_firstkey()
             if not key:
                 break
-            items = pypy_jit_codemap_del(key)
+            items = pypy_jit_codemap_del(key, 1)
             lltype.free(items, flavor='raw', track_allocation=False)
 
     def free_asm_block(self, start, stop):
-        items = pypy_jit_codemap_del(start)
+        items = pypy_jit_codemap_del(start, stop - start)
         if items:
             lltype.free(items, flavor='raw', track_allocation=False)
         pypy_jit_depthmap_clear(start, stop - start)
diff --git a/rpython/jit/backend/llsupport/src/codemap.c b/rpython/jit/backend/llsupport/src/codemap.c
--- a/rpython/jit/backend/llsupport/src/codemap.c
+++ b/rpython/jit/backend/llsupport/src/codemap.c
@@ -1,4 +1,5 @@
 #include "src/precommondefs.h"
+#include 
 
 #ifndef HAS_SKIPLIST
 # error "skiplist.c needs to be included before"
@@ -51,17 +52,26 @@
 }
 
 RPY_EXTERN
-long *pypy_jit_codemap_del(uintptr_t addr)
+long *pypy_jit_codemap_del(uintptr_t addr, unsigned int size)
 {
+    uintptr_t search_key = addr + size - 1;
     long *result;
     skipnode_t *node;
 
+    /* There should be either zero or one codemap entry in the range.
+       In theory it should take the complete range, but for alignment
+       reasons the [addr, addr+size] range can be slightly bigger. */
+    node = skiplist_search(&jit_codemap_head, search_key);
+    if (node->key < addr)
+        return NULL;
+
     pypy_codemap_invalid_set(1);
-    node = skiplist_remove(&jit_codemap_head, addr);
+    skiplist_remove(&jit_codemap_head, node->key);
     pypy_codemap_invalid_set(0);
 
-    if (node == NULL)
-        return NULL;
+    /* there should be at most one */
+    assert(skiplist_search(&jit_codemap_head, search_key)->key < addr);
+
     result = ((codemap_data_t *)node->data)->bytecode_info;
     free(node);
     return result;
diff --git a/rpython/jit/backend/llsupport/test/test_codemap.py b/rpython/jit/backend/llsupport/test/test_codemap.py
--- a/rpython/jit/backend/llsupport/test/test_codemap.py
+++ b/rpython/jit/backend/llsupport/test/test_codemap.py
@@ -91,3 +91,17 @@
     codemap.free_asm_block(200, 300)
     assert unpack_traceback(225) == []
     codemap.free()
+
+def test_free_with_alignment():
+    codemap = CodemapStorage()
+    codemap.setup()
+    builder = CodemapBuilder()
+    builder.enter_portal_frame(23, 34, 0)
+    builder.enter_portal_frame(45, 56, 20)
+    codemap.register_codemap(builder.get_final_bytecode(200, 100))
+    assert unpack_traceback(215) == [34]
+    assert unpack_traceback(225) == [34, 56]
+    codemap.free_asm_block(190, 310)   # a bit larger
+    assert unpack_traceback(215) == []
+    assert unpack_traceback(225) == []
+    codemap.free()

From noreply at buildbot.pypy.org  Mon Apr 20 17:32:26 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 20 Apr 2015 17:32:26 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof2: minor clean-ups
Message-ID: <20150420153226.D9FE81C08AF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: vmprof2
Changeset: r76851:ed7e507f2795
Date: 2015-04-20 17:11 +0200
http://bitbucket.org/pypy/pypy/changeset/ed7e507f2795/

Log:	minor clean-ups

diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py
--- a/rpython/jit/backend/llsupport/codemap.py
+++ b/rpython/jit/backend/llsupport/codemap.py
@@ -143,7 +143,6 @@
     def __init__(self):
         self.l = []
         self.patch_position = []
-        self.last_call_depth = -1
 
     def enter_portal_frame(self, jd_id, unique_id, relpos):
         self.l.append(unique_id)
@@ -159,7 +158,6 @@
 
     def inherit_code_from_position(self, pos):
         lst = unpack_traceback(pos)
-        self.last_call_depth = len(lst) - 1
         for item in lst:
             self.l.append(item)
             self.l.append(0)
diff --git a/rpython/jit/backend/llsupport/src/codemap.c b/rpython/jit/backend/llsupport/src/codemap.c
--- a/rpython/jit/backend/llsupport/src/codemap.c
+++ b/rpython/jit/backend/llsupport/src/codemap.c
@@ -194,7 +194,7 @@
         return -1;
 
     rel_addr = (uintptr_t)loc - depthmap->key;
-    data = (codemap_data_t *)depthmap->data;
+    data = (depthmap_data_t *)depthmap->data;
     if (rel_addr >= data->block_size)
         return -1;
 

From noreply at buildbot.pypy.org  Mon Apr 20 21:36:34 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 20 Apr 2015 21:36:34 +0200 (CEST)
Subject: [pypy-commit] pypy callfamily: move these 2 methods to a more
	logical location
Message-ID: <20150420193634.23B4E1C0365@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: callfamily
Changeset: r76853:a3e80a13f169
Date: 2015-04-20 20:36 +0100
http://bitbucket.org/pypy/pypy/changeset/a3e80a13f169/

Log:	move these 2 methods to a more logical location

diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py
--- a/rpython/annotator/description.py
+++ b/rpython/annotator/description.py
@@ -53,6 +53,22 @@
             table.append(row)
             self.total_calltable_size += 1
 
+    def find_row(self, bookkeeper, descs, args, op):
+        shape = rawshape(args)
+        with bookkeeper.at_position(None):
+            row = build_calltable_row(descs, args, op)
+        index = self.calltable_lookup_row(shape, row)
+        return shape, index
+
+def build_calltable_row(descs, args, op):
+    # see comments in CallFamily
+    row = {}
+    for desc in descs:
+        graph = desc.get_graph(args, op)
+        assert isinstance(graph, FunctionGraph)
+        row[desc.rowkey()] = graph
+    return row
+
 
 class FrozenAttrFamily(object):
     """A family of FrozenDesc objects that have any common 'getattr' sites.
@@ -352,33 +368,15 @@
 
     @staticmethod
     def consider_call_site(descs, args, s_result, op):
+        family = descs[0].getcallfamily()
         shape = rawshape(args)
-        row = FunctionDesc.row_to_consider(descs, args, op)
-        family = descs[0].getcallfamily()
+        row = build_calltable_row(descs, args, op)
         family.calltable_add_row(shape, row)
         descs[0].mergecallfamilies(*descs[1:])
 
-    @staticmethod
-    def variant_for_call_site(bookkeeper, family, descs, args, op):
-        shape = rawshape(args)
-        with bookkeeper.at_position(None):
-            row = FunctionDesc.row_to_consider(descs, args, op)
-        index = family.calltable_lookup_row(shape, row)
-        return shape, index
-
     def rowkey(self):
         return self
 
-    @staticmethod
-    def row_to_consider(descs, args, op):
-        # see comments in CallFamily
-        row = {}
-        for desc in descs:
-            graph = desc.get_graph(args, op)
-            assert isinstance(graph, FunctionGraph)
-            row[desc.rowkey()] = graph
-        return row
-
     def get_s_signatures(self, shape):
         family = self.getcallfamily()
         table = family.calltables.get(shape)
@@ -923,7 +921,7 @@
     def consider_call_site(descs, args, s_result, op):
         cnt, keys, star = rawshape(args)
         shape = cnt + 1, keys, star  # account for the extra 'self'
-        row = FunctionDesc.row_to_consider(descs, args, op)
+        row = build_calltable_row(descs, args, op)
         family = descs[0].getcallfamily()
         family.calltable_add_row(shape, row)
         descs[0].mergecallfamilies(*descs[1:])
@@ -1093,7 +1091,7 @@
     def consider_call_site(descs, args, s_result, op):
         cnt, keys, star = rawshape(args)
         shape = cnt + 1, keys, star  # account for the extra 'self'
-        row = FunctionDesc.row_to_consider(descs, args, op)
+        row = build_calltable_row(descs, args, op)
         family = descs[0].getcallfamily()
         family.calltable_add_row(shape, row)
         descs[0].mergecallfamilies(*descs[1:])
diff --git a/rpython/rtyper/rpbc.py b/rpython/rtyper/rpbc.py
--- a/rpython/rtyper/rpbc.py
+++ b/rpython/rtyper/rpbc.py
@@ -321,9 +321,8 @@
     def get_concrete_llfn(self, s_pbc, args_s, op):
         bk = self.rtyper.annotator.bookkeeper
         descs = list(s_pbc.descriptions)
-        vfcs = FunctionDesc.variant_for_call_site
         args = simple_args(args_s)
-        shape, index = vfcs(bk, self.callfamily, descs, args, op)
+        shape, index = self.callfamily.find_row(bk, descs, args, op)
         funcdesc, = descs
         row_of_one_graph = self.callfamily.calltables[shape][index]
         graph = row_of_one_graph[funcdesc]
@@ -341,8 +340,7 @@
         args = hop.spaceop.build_args(hop.args_s[1:])
         s_pbc = hop.args_s[0]   # possibly more precise than self.s_pbc
         descs = list(s_pbc.descriptions)
-        vfcs = FunctionDesc.variant_for_call_site
-        shape, index = vfcs(bk, self.callfamily, descs, args, hop.spaceop)
+        shape, index = self.callfamily.find_row(bk, descs, args, hop.spaceop)
         row_of_graphs = self.callfamily.calltables[shape][index]
         anygraph = row_of_graphs.itervalues().next()  # pick any witness
         vfn = hop.inputarg(self, arg=0)
@@ -475,8 +473,7 @@
         args = hop.spaceop.build_args(hop.args_s[1:])
         s_pbc = hop.args_s[0]   # possibly more precise than self.s_pbc
         descs = list(s_pbc.descriptions)
-        vfcs = FunctionDesc.variant_for_call_site
-        shape, index = vfcs(bk, self.callfamily, descs, args, hop.spaceop)
+        shape, index = self.callfamily.find_row(bk, descs, args, hop.spaceop)
         row_of_graphs = self.callfamily.calltables[shape][index]
         anygraph = row_of_graphs.itervalues().next()  # pick any witness
         vlist = [hop.inputarg(self, arg=0)]

From noreply at buildbot.pypy.org  Tue Apr 21 08:36:29 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 21 Apr 2015 08:36:29 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c7: Add TransactionQueue.add_generator()
Message-ID: <20150421063629.5F74A1C04A1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r76854:2e010e0f1fdc
Date: 2015-04-21 08:36 +0200
http://bitbucket.org/pypy/pypy/changeset/2e010e0f1fdc/

Log:	Add TransactionQueue.add_generator()

diff --git a/lib_pypy/pypy_test/test_transaction.py b/lib_pypy/pypy_test/test_transaction.py
--- a/lib_pypy/pypy_test/test_transaction.py
+++ b/lib_pypy/pypy_test/test_transaction.py
@@ -47,6 +47,19 @@
             print lsts
         assert lsts == (range(10),) * 5, lsts
 
+def test_add_generator():
+    lst = []
+    def do_stuff(n):
+        for i in range(n):
+            lst.append(i)
+            yield
+    tq = transaction.TransactionQueue()
+    tq.add_generator(do_stuff(10))
+    tq.run()
+    if VERBOSE:
+        print lst
+    assert lst == range(10), lst
+
 def test_raise():
     class FooError(Exception):
         pass
diff --git a/lib_pypy/transaction.py b/lib_pypy/transaction.py
--- a/lib_pypy/transaction.py
+++ b/lib_pypy/transaction.py
@@ -128,6 +128,19 @@
         #   thread-local list.
         self._pending.append((f, args, kwds))
 
+    def add_generator(self, generator_iterator):
+        """Register N new transactions to be done by a generator-iterator
+        object.  Each 'yield' marks the limit of transactions.
+        """
+        def transact_until_yield():
+            # Ask for the next item in this transaction.  If we get it,
+            # then the 'for' loop below adds this function again and
+            # returns.
+            for ignored_yielded_value in generator_iterator:
+                self.add(transact_until_yield)
+                return
+        self.add(transact_until_yield)
+
     def run(self, nb_segments=0):
         """Run all transactions, and all transactions started by these
         ones, recursively, until the queue is empty.  If one transaction

From noreply at buildbot.pypy.org  Tue Apr 21 13:20:20 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 21 Apr 2015 13:20:20 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: fixes
Message-ID: <20150421112020.4CA331C0163@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1773:f581837ca590
Date: 2015-04-19 17:16 +0200
http://bitbucket.org/cffi/cffi/changeset/f581837ca590/

Log:	fixes

diff --git a/new/recompiler.py b/new/recompiler.py
--- a/new/recompiler.py
+++ b/new/recompiler.py
@@ -441,7 +441,7 @@
                 spaces = " " * len(fldname)
                 if (isinstance(fldtype, model.ArrayType) and
                        fldtype.length is None):
-                    size = '-1'
+                    size = '(size_t)-1'
                 else:
                     size = 'sizeof(((%s)0)->%s)' % (tp.get_c_name('*'), fldname)
                 c_field.append(
diff --git a/new/test_verify1.py b/new/test_verify1.py
--- a/new/test_verify1.py
+++ b/new/test_verify1.py
@@ -1,5 +1,5 @@
 import sys, math, py
-from cffi1 import FFI, VerificationError, model
+from cffi1 import FFI, VerificationError, VerificationMissing, model
 
 lib_m = ['m']
 if sys.platform == 'win32':

From noreply at buildbot.pypy.org  Tue Apr 21 13:20:21 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 21 Apr 2015 13:20:21 +0200 (CEST)
Subject: [pypy-commit] cffi cffi-1.0: Careful logic to support "unnamed"
 structs at the same place as before.
Message-ID: <20150421112021.5627A1C0163@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: cffi-1.0
Changeset: r1774:8f7f19808230
Date: 2015-04-21 13:20 +0200
http://bitbucket.org/cffi/cffi/changeset/8f7f19808230/

Log:	Careful logic to support "unnamed" structs at the same place as
	before.

diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -19,6 +19,8 @@
     basestring = str
 
 
+FFIError = _cffi1_backend.FFI.error
+
 class CDefError(Exception):
     def __str__(self):
         try:
diff --git a/new/realize_c_type.c b/new/realize_c_type.c
--- a/new/realize_c_type.c
+++ b/new/realize_c_type.c
@@ -307,24 +307,38 @@
             strcat(name, s->name);
             x = new_struct_or_union_type(name, flags);
 
+            CTypeDescrObject *ct = NULL;
             if (s->first_field_index >= 0) {
-                CTypeDescrObject *ct = (CTypeDescrObject *)x;
-                ct->ct_size = s->size;
+                ct = (CTypeDescrObject *)x;
+                ct->ct_size = (Py_ssize_t)s->size;
                 ct->ct_length = s->alignment;
                 ct->ct_flags &= ~CT_IS_OPAQUE;
                 ct->ct_flags |= CT_LAZY_FIELD_LIST;
                 ct->ct_extra = builder;
             }
 
-            /* We are going to update the "primary" OP_STRUCT_OR_UNION
-               slot below, which may be the same or a different one as
-               the "current" slot.  If it is a different one, the
-               current slot is not updated.  But in this case, the
-               next time we walk the same current slot, we'll find the
-               'x' object in the primary slot (op2, above) and then we
-               will update the current slot. */
-            opcodes = builder->ctx.types;
-            index = s->type_index;
+            /* Update the "primary" OP_STRUCT_OR_UNION slot, which
+               may be the same or a different slot than the "current" one */
+            assert((((uintptr_t)x) & 1) == 0);
+            assert(builder->ctx.types[s->type_index] == op2);
+            Py_INCREF(x);
+            builder->ctx.types[s->type_index] = x;
+
+            if (s->size == (size_t)-2) {
+                /* oops, this struct is unnamed and we couldn't generate
+                   a C expression to get its size.  We have to rely on
+                   complete_struct_or_union() to compute it now. */
+                assert(ct != NULL);
+                if (do_realize_lazy_struct(ct) < 0) {
+                    builder->ctx.types[s->type_index] = op2;
+                    return NULL;
+                }
+            }
+
+            /* Done, leave without updating the "current" slot because
+               it may be done already above.  If not, never mind, the
+               next call to realize_c_type() will do it. */
+            return x;
         }
         break;
     }
@@ -451,7 +465,12 @@
                 return -1;
             }
 
-            if (detect_custom_layout(ct, SF_STD_FIELD_POS,
+            if (fld->field_offset == (size_t)-1) {
+                /* unnamed struct, with field positions and sizes entirely
+                   determined by complete_struct_or_union() and not checked */
+                assert(fld->field_size == -1);
+            }
+            else if (detect_custom_layout(ct, SF_STD_FIELD_POS,
                                      ctf->ct_size, fld->field_size,
                                      "wrong size for field '",
                                      fld->name, "'") < 0)
diff --git a/new/recompiler.py b/new/recompiler.py
--- a/new/recompiler.py
+++ b/new/recompiler.py
@@ -70,6 +70,10 @@
             self._typesdict[tp] = None
             if isinstance(tp, model.FunctionPtrType):
                 self._do_collect_type(tp.as_raw_function())
+            elif isinstance(tp, model.StructOrUnion):
+                if tp.fldtypes is not None:
+                    for name1, tp1 in zip(tp.fldnames, tp.fldtypes):
+                        self._do_collect_type(self._field_type(tp, name1, tp1))
             else:
                 for _, x in tp._get_items():
                     self._do_collect_type(x)
@@ -118,6 +122,7 @@
         #
         # the declaration of '_cffi_types'
         prnt('static void *_cffi_types[] = {')
+        self.cffi_types = tuple(self.cffi_types)    # don't change any more
         typeindex2type = dict([(i, tp) for (tp, i) in self._typesdict.items()])
         for i, op in enumerate(self.cffi_types):
             comment = ''
@@ -139,7 +144,9 @@
         self._lsts = {}
         for step_name in ALL_STEPS:
             self._lsts[step_name] = []
+        self._seen_struct_unions = set()
         self._generate("ctx")
+        self._add_missing_struct_unions()
         for step_name in ALL_STEPS:
             lst = self._lsts[step_name]
             nums[step_name] = len(lst)
@@ -284,9 +291,9 @@
         self._lsts["typename"].append(
             '  { "%s", %d },' % (name, type_index))
         if getattr(tp, "origin", None) == "unknown_type":
-            self._generate_cpy_struct_ctx(tp, tp.name)
+            self._struct_ctx(tp, tp.name, approxname=None)
         elif isinstance(tp, model.NamedPointerType):
-            self._generate_cpy_struct_ctx(tp.totype, tp.totype.name)
+            self._struct_ctx(tp.totype, tp.totype.name, approxname=None)
 
     # ----------
     # function declarations
@@ -393,18 +400,14 @@
             tp_field = tp_field.resolve_length(actual_length)
         return tp_field
 
-    def _generate_cpy_struct_collecttype(self, tp, name):
+    def _struct_collecttype(self, tp):
         self._do_collect_type(tp)
-        if tp.fldtypes is not None:
-            for name1, tp1 in zip(tp.fldnames, tp.fldtypes):
-                self._do_collect_type(self._field_type(tp, name1, tp1))
 
-    def _generate_cpy_struct_decl(self, tp, name):
+    def _struct_decl(self, tp, cname, approxname):
         if tp.fldtypes is None:
             return
-        cname = ('%s %s' % (tp.kind, name)).strip()
         prnt = self._prnt
-        checkfuncname = '_cffi_checkfld_%s' % (name,)
+        checkfuncname = '_cffi_checkfld_%s' % (approxname,)
         prnt('__attribute__((unused))')
         prnt('static void %s(%s *p)' % (checkfuncname, cname))
         prnt('{')
@@ -423,10 +426,10 @@
                 except ffiplatform.VerificationError as e:
                     prnt('  /* %s */' % str(e))   # cannot verify it, ignore
         prnt('}')
-        prnt('struct _cffi_align_%s { char x; %s y; };' % (name, cname))
+        prnt('struct _cffi_align_%s { char x; %s y; };' % (approxname, cname))
         prnt()
 
-    def _generate_cpy_struct_ctx(self, tp, name):
+    def _struct_ctx(self, tp, cname, approxname):
         type_index = self._typesdict[tp]
         flags = []
         if tp.partial:
@@ -435,45 +438,104 @@
             flags.append('CT_UNION')
         flags = ('|'.join(flags)) or '0'
         if tp.fldtypes is not None:
-            c_field = [name]
+            c_field = [approxname]
             for fldname, fldtype in zip(tp.fldnames, tp.fldtypes):
                 fldtype = self._field_type(tp, fldname, fldtype)
                 spaces = " " * len(fldname)
-                if (isinstance(fldtype, model.ArrayType) and
-                       fldtype.length is None):
+                # cname is None for _add_missing_struct_unions() only
+                if cname is None or (
+                        isinstance(fldtype, model.ArrayType) and
+                        fldtype.length is None):
                     size = '(size_t)-1'
                 else:
                     size = 'sizeof(((%s)0)->%s)' % (tp.get_c_name('*'), fldname)
+                if cname is None:
+                    offset = '(size_t)-1'
+                else:
+                    offset = 'offsetof(%s, %s)' % (tp.get_c_name(''), fldname)
                 c_field.append(
-                    '  { "%s", offsetof(%s, %s),\n' % (
-                            fldname, tp.get_c_name(''), fldname) +
+                    '  { "%s", %s,\n' % (fldname, offset) +
                     '     %s   %s,\n' % (spaces, size) +
                     '     %s   _CFFI_OP(_CFFI_OP_NOOP, %s) },' % (
                             spaces, self._typesdict[fldtype]))
             self._lsts["field"].append('\n'.join(c_field))
-            size_align = ('\n' +
-                '    sizeof(%s %s),\n' % (tp.kind, name) +
-                '    offsetof(struct _cffi_align_%s, y),\n' % (name,) +
-                '    _cffi_FIELDS_FOR_%s, %d },' % (name, len(tp.fldtypes),))
+            #
+            if cname is None:  # unknown name, for _add_missing_struct_unions
+                size_align = (' (size_t)-2, -2, /* unnamed */\n' +
+                    '    _cffi_FIELDS_FOR_%s, %d },' % (approxname,
+                                                        len(tp.fldtypes),))
+            else:
+                size_align = ('\n' +
+                    '    sizeof(%s),\n' % (cname,) +
+                    '    offsetof(struct _cffi_align_%s, y),\n'% (approxname,) +
+                    '    _cffi_FIELDS_FOR_%s, %d },' % (approxname,
+                                                        len(tp.fldtypes),))
         else:
             size_align = ' (size_t)-1, -1, -1, 0 /* opaque */ },'
         self._lsts["struct_union"].append(
-            '  { "%s", %d, %s,' % (name, type_index, flags) + size_align)
+            '  { "%s", %d, %s,' % (tp.name, type_index, flags) + size_align)
+        self._seen_struct_unions.add(tp)
+
+    def _add_missing_struct_unions(self):
+        # not very nice, but some struct declarations might be missing
+        # because they don't have any known C name.  Check that they are
+        # not partial (we can't complete or verify them!) and emit them
+        # anonymously.
+        for tp in list(self._struct_unions):
+            if tp not in self._seen_struct_unions:
+                if tp.partial:
+                    raise NotImplementedError("internal inconsistency: %r is "
+                                              "partial but was not seen at "
+                                              "this point" % (tp,))
+                assert tp.name.startswith('$') and tp.name[1:].isdigit()
+                self._struct_ctx(tp, None, tp.name[1:])
 
     def _fix_final_field_list(self, lst):
         count = 0
         for i in range(len(lst)):
             struct_fields = lst[i]
-            name = struct_fields.split('\n')[0]
-            define_macro = '#define _cffi_FIELDS_FOR_%s  %d' % (name, count)
-            lst[i] = define_macro + struct_fields[len(name):]
+            pname = struct_fields.split('\n')[0]
+            define_macro = '#define _cffi_FIELDS_FOR_%s  %d' % (pname, count)
+            lst[i] = define_macro + struct_fields[len(pname):]
             count += lst[i].count('\n  { "')
 
+    def _generate_cpy_struct_collecttype(self, tp, name):
+        self._struct_collecttype(tp)
     _generate_cpy_union_collecttype = _generate_cpy_struct_collecttype
+
+    def _generate_cpy_struct_decl(self, tp, name):
+        cname = tp.get_c_name('')
+        self._struct_decl(tp, cname, cname.replace(' ', '_'))
     _generate_cpy_union_decl = _generate_cpy_struct_decl
+
+    def _generate_cpy_struct_ctx(self, tp, name, prefix='s'):
+        cname = tp.get_c_name('')
+        self._struct_ctx(tp, cname, cname.replace(' ', '_'))
     _generate_cpy_union_ctx = _generate_cpy_struct_ctx
 
     # ----------
+    # 'anonymous' declarations.  These are produced for anonymous structs
+    # or unions; the 'name' is obtained by a typedef.
+
+    def _generate_cpy_anonymous_collecttype(self, tp, name):
+        if isinstance(tp, model.EnumType):
+            self._generate_cpy_enum_collecttype(tp, name)
+        else:
+            self._struct_collecttype(tp)
+
+    def _generate_cpy_anonymous_decl(self, tp, name):
+        if isinstance(tp, model.EnumType):
+            self._generate_cpy_enum_decl(tp, name, '')
+        else:
+            self._struct_decl(tp, name, 'typedef_' + name)
+
+    def _generate_cpy_anonymous_ctx(self, tp, name):
+        if isinstance(tp, model.EnumType):
+            self._generate_cpy_enum_ctx(tp, name, '')
+        else:
+            self._struct_ctx(tp, name, 'typedef_' + name)
+
+    # ----------
     # constants, declared with "static const ..."
 
     def _generate_cpy_const(self, is_int, name, tp=None, category='const',
diff --git a/new/test_recompiler.py b/new/test_recompiler.py
--- a/new/test_recompiler.py
+++ b/new/test_recompiler.py
@@ -83,6 +83,10 @@
                      "(PRIMITIVE 7)(PRIMITIVE 9)"
                      "(STRUCT_UNION 0)")
 
+def test_anonymous_struct_with_typedef():
+    check_type_table("typedef struct { int a; long b; } foo_t;",
+                     "(STRUCT_UNION 0)(PRIMITIVE 7)(PRIMITIVE 9)")
+
 
 def test_math_sin():
     import math
@@ -291,3 +295,19 @@
     # 'x' is another  object on lib, made very indirectly
     x = type(lib).__dir__.__get__(lib)
     py.test.raises(TypeError, ffi.typeof, x)
+
+def test_verify_anonymous_struct_with_typedef():
+    ffi = FFI()
+    ffi.cdef("typedef struct { int a; long b; ...; } foo_t;")
+    verify(ffi, 'test_verify_anonymous_struct_with_typedef',
+           "typedef struct { long b; int hidden, a; } foo_t;")
+    p = ffi.new("foo_t *", {'b': 42})
+    assert p.b == 42
+
+def test_verify_anonymous_struct_with_star_typedef():
+    ffi = FFI()
+    ffi.cdef("typedef struct { int a; long b; } *foo_t;")
+    verify(ffi, 'test_verify_anonymous_struct_with_star_typedef',
+           "typedef struct { int a; long b; } *foo_t;")
+    p = ffi.new("foo_t", {'b': 42})
+    assert p.b == 42

From noreply at buildbot.pypy.org  Tue Apr 21 13:35:04 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 21 Apr 2015 13:35:04 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof2: merge default
Message-ID: <20150421113504.C954E1C06FD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof2
Changeset: r76855:cb3455f7a86e
Date: 2015-04-21 13:34 +0200
http://bitbucket.org/pypy/pypy/changeset/cb3455f7a86e/

Log:	merge default

diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -129,7 +129,8 @@
     # This is meant to be a *mixin*.
 
     def is_data_descr(space, w_obj):
-        return space.lookup(w_obj, '__set__') is not None
+        return (space.lookup(w_obj, '__set__') is not None or
+                space.lookup(w_obj, '__delete__') is not None)
 
     def get_and_call_args(space, w_descr, w_obj, args):
         # a special case for performance and to avoid infinite recursion
diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py
--- a/pypy/objspace/test/test_descroperation.py
+++ b/pypy/objspace/test/test_descroperation.py
@@ -607,6 +607,18 @@
 
         raises(AttributeError, lambda: A().a)
 
+    def test_delete_descriptor(self):
+        class Prop(object):
+            def __get__(self, obj, cls):
+                return 42
+            def __delete__(self, obj):
+                obj.deleted = True
+        class C(object):
+            x = Prop()
+        obj = C()
+        del obj.x
+        assert obj.deleted
+
     def test_non_callable(self):
         meth = classmethod(1).__get__(1)
         raises(TypeError, meth)
diff --git a/pypy/tool/gdb_pypy.py b/pypy/tool/gdb_pypy.py
--- a/pypy/tool/gdb_pypy.py
+++ b/pypy/tool/gdb_pypy.py
@@ -5,7 +5,7 @@
 
 Or, alternatively:
 
-(gdb) python execfile('/path/to/gdb_pypy.py')
+(gdb) python exec(open('/path/to/gdb_pypy.py').read())
 """
 
 import re
@@ -55,10 +55,10 @@
 
 class RPyType(Command):
     """
-    Prints the RPython type of the expression (remember to dereference it!)
+    Prints the RPython type of the expression.
     E.g.:
 
-    (gdb) rpy_type *l_v123
+    (gdb) rpy_type l_v123
     GcStruct pypy.foo.Bar { super, inst_xxx, inst_yyy }
     """
 
@@ -73,21 +73,30 @@
 
     def invoke(self, arg, from_tty):
         # some magic code to automatically reload the python file while developing
-        from pypy.tool import gdb_pypy
         try:
-            reload(gdb_pypy)
+            from pypy.tool import gdb_pypy
+            try:
+                reload(gdb_pypy)
+            except:
+                import imp
+                imp.reload(gdb_pypy)
+            gdb_pypy.RPyType.prog2typeids = self.prog2typeids # persist the cache
+            self.__class__ = gdb_pypy.RPyType
+            result = self.do_invoke(arg, from_tty)
+            if not isinstance(result, str):
+                result = result.decode('latin-1')
+            print(result)
         except:
-            import imp
-            imp.reload(gdb_pypy)
-        gdb_pypy.RPyType.prog2typeids = self.prog2typeids # persist the cache
-        self.__class__ = gdb_pypy.RPyType
-        print (self.do_invoke(arg, from_tty).decode('latin-1'))
+            import traceback
+            traceback.print_exc()
 
     def do_invoke(self, arg, from_tty):
         try:
             offset = int(arg)
         except ValueError:
             obj = self.gdb.parse_and_eval(arg)
+            if obj.type.code == self.gdb.TYPE_CODE_PTR:
+                obj = obj.dereference()
             hdr = lookup(obj, '_gcheader')
             tid = hdr['h_tid']
             if sys.maxsize < 2**32:
@@ -100,7 +109,7 @@
         if offset in typeids:
             return typeids[offset]
         else:
-            return 'Cannot find the type with offset %d' % offset
+            return 'Cannot find the type with offset 0x%x' % offset
 
     def get_typeids(self):
         try:
@@ -127,7 +136,7 @@
         try:
             self.gdb.execute('dump binary memory %s %s %s+%d' %
                              (fname, vstart, vstart, length))
-            with open(fname, 'rt') as fobj:
+            with open(fname, 'rb') as fobj:
                 data = fobj.read()
             return TypeIdsMap(zlib.decompress(data).splitlines(True), self.gdb)
         finally:
@@ -242,6 +251,8 @@
     fields
     """
 
+    recursive = False
+
     def __init__(self, val):
         self.val = val
 
@@ -249,29 +260,47 @@
     def lookup(cls, val, gdb=None):
         t = val.type
         if (is_ptr(t, gdb) and t.target().tag is not None and
-            re.match(r'pypy_list\d*', t.target().tag)):
+            re.match(r'pypy_(list|array)\d*', t.target().tag)):
             return cls(val)
         return None
 
     def to_string(self):
-        length = int(self.val['l_length'])
-        array = self.val['l_items']
-        allocated = int(array['length'])
-        items = array['items']
-        itemlist = []
-        for i in range(length):
-            item = items[i]
-            itemlist.append(str(item))
-        str_items = ', '.join(itemlist)
-        return 'r[%s] (len=%d, alloc=%d)' % (str_items, length, allocated)
+        t = self.val.type
+        if t.target().tag.startswith(r'pypy_array'):
+            if not self.val:
+                return 'r(null_array)'
+            length = int(self.val['length'])
+            items = self.val['items']
+            allocstr = ''
+        else:
+            if not self.val:
+                return 'r(null_list)'
+            length = int(self.val['l_length'])
+            array = self.val['l_items']
+            allocated = int(array['length'])
+            items = array['items']
+            allocstr = ', alloc=%d' % allocated
+        if RPyListPrinter.recursive:
+            str_items = '...'
+        else:
+            RPyListPrinter.recursive = True
+            try:
+                itemlist = []
+                for i in range(length):
+                    item = items[i]
+                    itemlist.append(str(item))    # may recurse here
+                str_items = ', '.join(itemlist)
+            finally:
+                RPyListPrinter.recursive = False
+        return 'r[%s] (len=%d%s)' % (str_items, length, allocstr)
 
 
 try:
     import gdb
     RPyType() # side effects
-    gdb.pretty_printers += [
+    gdb.pretty_printers = [
         RPyStringPrinter.lookup,
         RPyListPrinter.lookup
-        ]
+        ] + gdb.pretty_printers
 except ImportError:
     pass
diff --git a/pypy/tool/test/test_gdb_pypy.py b/pypy/tool/test/test_gdb_pypy.py
--- a/pypy/tool/test/test_gdb_pypy.py
+++ b/pypy/tool/test/test_gdb_pypy.py
@@ -204,6 +204,22 @@
     mylist.type.target().tag = None
     assert gdb_pypy.RPyListPrinter.lookup(mylist, FakeGdb) is None
 
+def test_pprint_array():
+    d = {'_gcheder': {'h_tid': 234}, 'length': 3, 'items': [20, 21, 22]}
+    mylist = PtrValue(d, type_tag='pypy_array1')
+    printer = gdb_pypy.RPyListPrinter.lookup(mylist, FakeGdb)
+    assert printer.to_string() == 'r[20, 21, 22] (len=3)'
+
+def test_pprint_null_list():
+    mylist = PtrValue({}, type_tag='pypy_list1')
+    printer = gdb_pypy.RPyListPrinter.lookup(mylist, FakeGdb)
+    assert printer.to_string() == 'r(null_list)'
+
+def test_pprint_null_array():
+    mylist = PtrValue({}, type_tag='pypy_array1')
+    printer = gdb_pypy.RPyListPrinter.lookup(mylist, FakeGdb)
+    assert printer.to_string() == 'r(null_array)'
+
 def test_typeidsmap():
     gdb = FakeGdb('', {exprmember(1): 111,
                        exprmember(2): 222,
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -1972,7 +1972,10 @@
     def rewrite_op_jit_force_virtualizable(self, op):
         # this one is for virtualizables
         vinfo = self.get_vinfo(op.args[0])
-        assert vinfo is not None
+        assert vinfo is not None, (
+            "%r is a class with _virtualizable_, but no jitdriver was found"
+            " with a 'virtualizable' argument naming a variable of that class"
+            % op.args[0].concretetype)
         self.vable_flags[op.args[0]] = op.args[2].value
         return []
 

From noreply at buildbot.pypy.org  Tue Apr 21 13:51:35 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 21 Apr 2015 13:51:35 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof2: fix the llgraph runner
Message-ID: <20150421115135.570FD1C0EFC@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof2
Changeset: r76856:b0b2d41618d1
Date: 2015-04-21 13:51 +0200
http://bitbucket.org/pypy/pypy/changeset/b0b2d41618d1/

Log:	fix the llgraph runner

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -245,8 +245,8 @@
         self.stats = stats or MiniStats()
         self.vinfo_for_tests = kwds.get('vinfo_for_tests', None)
 
-    def compile_loop(self, inputargs, operations, looptoken, log=True,
-                     name='', logger=None):
+    def compile_loop(self, jd_id, unique_id, inputargs, operations, looptoken,
+                     log=True, name='', logger=None):
         clt = model.CompiledLoopToken(self, looptoken.number)
         looptoken.compiled_loop_token = clt
         lltrace = LLTrace(inputargs, operations)
@@ -1028,7 +1028,7 @@
     def execute_enter_portal_frame(self, descr, *args):
         pass
 
-    def execute_leave_portal_frame(self, descr):
+    def execute_leave_portal_frame(self, descr, *args):
         pass
 
     def execute_new_with_vtable(self, _, vtable):

From noreply at buildbot.pypy.org  Tue Apr 21 15:42:02 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue, 21 Apr 2015 15:42:02 +0200 (CEST)
Subject: [pypy-commit] pypy callfamily: extract method
Message-ID: <20150421134202.524651C12C0@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: callfamily
Changeset: r76857:436ecb10fb64
Date: 2015-04-20 23:51 +0100
http://bitbucket.org/pypy/pypy/changeset/436ecb10fb64/

Log:	extract method

diff --git a/rpython/rtyper/rpbc.py b/rpython/rtyper/rpbc.py
--- a/rpython/rtyper/rpbc.py
+++ b/rpython/rtyper/rpbc.py
@@ -438,6 +438,13 @@
         key = shape, index, tuple(argtypes), resulttype
         if key in self._dispatch_cache:
             return self._dispatch_cache[key]
+        graph = self.make_dispatcher(shape, index, argtypes, resulttype)
+        self.rtyper.annotator.translator.graphs.append(graph)
+        ll_ret = getfunctionptr(graph)
+        c_ret = self._dispatch_cache[key] = inputconst(typeOf(ll_ret), ll_ret)
+        return c_ret
+
+    def make_dispatcher(self, shape, index, argtypes, resulttype):
         inputargs = [varoftype(t) for t in [Char] + argtypes]
         startblock = Block(inputargs)
         startblock.exitswitch = inputargs[0]
@@ -462,11 +469,7 @@
             links.append(Link(inputargs[1:], b, chr(i)))
             links[-1].llexitcase = chr(i)
         startblock.closeblock(*links)
-        self.rtyper.annotator.translator.graphs.append(graph)
-        ll_ret = getfunctionptr(graph)
-        #FTYPE = FuncType
-        c_ret = self._dispatch_cache[key] = inputconst(typeOf(ll_ret), ll_ret)
-        return c_ret
+        return graph
 
     def call(self, hop):
         bk = self.rtyper.annotator.bookkeeper

From noreply at buildbot.pypy.org  Tue Apr 21 15:42:03 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue, 21 Apr 2015 15:42:03 +0200 (CEST)
Subject: [pypy-commit] pypy callfamily: simplify get_concrete_llfn()
Message-ID: <20150421134203.8AA6C1C12C0@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: callfamily
Changeset: r76858:4565986ec922
Date: 2015-04-21 03:31 +0100
http://bitbucket.org/pypy/pypy/changeset/4565986ec922/

Log:	simplify get_concrete_llfn()

diff --git a/rpython/rtyper/rpbc.py b/rpython/rtyper/rpbc.py
--- a/rpython/rtyper/rpbc.py
+++ b/rpython/rtyper/rpbc.py
@@ -320,12 +320,10 @@
 
     def get_concrete_llfn(self, s_pbc, args_s, op):
         bk = self.rtyper.annotator.bookkeeper
-        descs = list(s_pbc.descriptions)
+        funcdesc, = s_pbc.descriptions
         args = simple_args(args_s)
-        shape, index = self.callfamily.find_row(bk, descs, args, op)
-        funcdesc, = descs
-        row_of_one_graph = self.callfamily.calltables[shape][index]
-        graph = row_of_one_graph[funcdesc]
+        with bk.at_position(None):
+            graph = funcdesc.get_graph(args, op)
         llfn = self.rtyper.getcallable(graph)
         return inputconst(typeOf(llfn), llfn)
 

From noreply at buildbot.pypy.org  Tue Apr 21 18:21:44 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 21 Apr 2015 18:21:44 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof2: fix some signatures
Message-ID: <20150421162144.2FCC31C0163@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof2
Changeset: r76859:4d05d587af36
Date: 2015-04-21 18:21 +0200
http://bitbucket.org/pypy/pypy/changeset/4d05d587af36/

Log:	fix some signatures

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -245,8 +245,8 @@
         self.stats = stats or MiniStats()
         self.vinfo_for_tests = kwds.get('vinfo_for_tests', None)
 
-    def compile_loop(self, jd_id, unique_id, inputargs, operations, looptoken,
-                     log=True, name='', logger=None):
+    def compile_loop(self, inputargs, operations, looptoken, jd_id=0,
+                     unique_id=0, log=True, name='', logger=None):
         clt = model.CompiledLoopToken(self, looptoken.number)
         looptoken.compiled_loop_token = clt
         lltrace = LLTrace(inputargs, operations)
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -82,8 +82,8 @@
     def finish_once(self):
         self.codemap.finish_once()
 
-    def compile_loop(self, jd_id, unique_id, inputargs, operations, looptoken,
-                     log=True, name='', logger=None):
+    def compile_loop(self, inputargs, operations, looptoken, jd_id=0,
+                     unique_id=0, log=True, name='', logger=None):
         return self.assembler.assemble_loop(jd_id, unique_id, logger, name,
                                             inputargs, operations,
                                             looptoken, log=log)
diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py
--- a/rpython/jit/backend/model.py
+++ b/rpython/jit/backend/model.py
@@ -51,8 +51,8 @@
         """
         return False
 
-    def compile_loop(self, jd_id, unique_id, inputargs, operations, looptoken,
-                     log=True, name='', logger=None):
+    def compile_loop(self, inputargs, operations, looptoken, jd_id=0,
+                     unique_id=0, log=True, name='', logger=None):
         """Assemble the given loop.
         Should create and attach a fresh CompiledLoopToken to
         looptoken.compiled_loop_token and stick extra attributes
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -307,8 +307,9 @@
                     looptoken, log=True, name=''):
     metainterp_sd.logger_ops.log_loop(inputargs, operations, -2,
                                       'compiling', name=name)
-    return metainterp_sd.cpu.compile_loop(jd_id, unique_id, inputargs,
+    return metainterp_sd.cpu.compile_loop(inputargs,
                                           operations, looptoken,
+                                          jd_id=jd_id, unique_id=unique_id,
                                           log=log, name=name,
                                           logger=metainterp_sd.logger_ops)
 

From noreply at buildbot.pypy.org  Tue Apr 21 19:24:07 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 21 Apr 2015 19:24:07 +0200 (CEST)
Subject: [pypy-commit] pypy default: Detect objects with h_tid==-42
Message-ID: <20150421172407.7CDED1C127C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76860:e30bd43e438c
Date: 2015-04-21 19:24 +0200
http://bitbucket.org/pypy/pypy/changeset/e30bd43e438c/

Log:	Detect objects with h_tid==-42

diff --git a/pypy/tool/gdb_pypy.py b/pypy/tool/gdb_pypy.py
--- a/pypy/tool/gdb_pypy.py
+++ b/pypy/tool/gdb_pypy.py
@@ -99,6 +99,8 @@
                 obj = obj.dereference()
             hdr = lookup(obj, '_gcheader')
             tid = hdr['h_tid']
+            if tid == -42:      # forwarded?
+                return 'Forwarded'
             if sys.maxsize < 2**32:
                 offset = tid & 0xFFFF     # 32bit
             else:

From noreply at buildbot.pypy.org  Tue Apr 21 20:01:46 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 21 Apr 2015 20:01:46 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof2: kill this test,
	the logic is now mostly trivial
Message-ID: <20150421180146.6F7891C0541@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof2
Changeset: r76861:6dfc744df435
Date: 2015-04-21 20:01 +0200
http://bitbucket.org/pypy/pypy/changeset/6dfc744df435/

Log:	kill this test, the logic is now mostly trivial

diff --git a/rpython/jit/backend/llsupport/test/test_codemap.py b/rpython/jit/backend/llsupport/test/test_codemap.py
--- a/rpython/jit/backend/llsupport/test/test_codemap.py
+++ b/rpython/jit/backend/llsupport/test/test_codemap.py
@@ -55,43 +55,6 @@
     assert stack_depth_at_loc(10) == 9
     codemap.free()
 
-def test_codemaps():
-    builder = CodemapBuilder()
-    builder.debug_merge_point(0, 102, 0)
-    builder.debug_merge_point(0, 102, 13)
-    builder.debug_merge_point(1, 104, 15)
-    builder.debug_merge_point(1, 104, 16)
-    builder.debug_merge_point(2, 106, 20)
-    builder.debug_merge_point(2, 106, 25)
-    builder.debug_merge_point(1, 104, 30)
-    builder.debug_merge_point(0, 102, 35)
-    codemap = CodemapStorage()
-    codemap.setup()
-    codemap.register_codemap(builder.get_final_bytecode(100, 40))
-    builder = CodemapBuilder()
-    builder.debug_merge_point(0, 202, 0)
-    builder.debug_merge_point(0, 202, 10)
-    builder.debug_merge_point(1, 204, 20)
-    builder.debug_merge_point(1, 204, 30)
-    builder.debug_merge_point(2, 206, 40)
-    builder.debug_merge_point(2, 206, 50)
-    builder.debug_merge_point(1, 204, 60)
-    builder.debug_merge_point(0, 202, 70)
-    codemap.register_codemap(builder.get_final_bytecode(200, 100))
-    assert unpack_traceback(110) == [102]
-    assert unpack_traceback(117) == [102, 104]
-    assert unpack_traceback(121) == [102, 104, 106]
-    assert unpack_traceback(131) == [102, 104]
-    assert unpack_traceback(137) == [102]
-    assert unpack_traceback(205) == [202]
-    assert unpack_traceback(225) == [202, 204]
-    assert unpack_traceback(245) == [202, 204, 206]
-    assert unpack_traceback(265) == [202, 204]
-    assert unpack_traceback(275) == [202]
-    codemap.free_asm_block(200, 300)
-    assert unpack_traceback(225) == []
-    codemap.free()
-
 def test_free_with_alignment():
     codemap = CodemapStorage()
     codemap.setup()

From noreply at buildbot.pypy.org  Tue Apr 21 20:03:09 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 21 Apr 2015 20:03:09 +0200 (CEST)
Subject: [pypy-commit] pypy vmprof2: ups fix that
Message-ID: <20150421180309.8B39F1C06FD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof2
Changeset: r76862:ba287c05cbb7
Date: 2015-04-21 20:03 +0200
http://bitbucket.org/pypy/pypy/changeset/ba287c05cbb7/

Log:	ups fix that

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -929,7 +929,7 @@
     ]
     operations[1].setfailargs([])
     operations = get_deep_immutable_oplist(operations)
-    cpu.compile_loop(0, -1, inputargs, operations, jitcell_token, log=False)
+    cpu.compile_loop(inputargs, operations, jitcell_token, log=False)
     if memory_manager is not None:    # for tests
         memory_manager.keep_loop_alive(jitcell_token)
     return jitcell_token

From noreply at buildbot.pypy.org  Wed Apr 22 11:22:31 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 22 Apr 2015 11:22:31 +0200 (CEST)
Subject: [pypy-commit] pypy object-dtype2: fix compile for
 __array_interface__ dictionary lookup
Message-ID: <20150422092231.C063D1C02A3@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: object-dtype2
Changeset: r76864:1b847ed40e52
Date: 2015-04-20 19:58 +0300
http://bitbucket.org/pypy/pypy/changeset/1b847ed40e52/

Log:	fix compile for __array_interface__ dictionary lookup

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -89,7 +89,8 @@
         return self.wrap(len(w_obj.items))
 
     def getattr(self, w_obj, w_attr):
-        return StringObject(NonConstant('foo'))
+        assert isinstance(w_attr, StringObject)
+        return w_obj.getdictvalue(self, w_attr.v)
 
     def isinstance_w(self, w_obj, w_tp):
         try:
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -45,6 +45,9 @@
         if e.match(space, space.w_AttributeError):
             return None
         raise
+    if w_interface is None:
+        # happens from compile.py
+        return None
     version = space.int_w(space.finditem(w_interface, space.wrap("version")))
     if version < 3:
         raise oefmt(space.w_NotImplementedError,

From noreply at buildbot.pypy.org  Wed Apr 22 11:22:30 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 22 Apr 2015 11:22:30 +0200 (CEST)
Subject: [pypy-commit] pypy object-dtype2: test, implement coerce_subtype
Message-ID: <20150422092230.72F5C1C02A3@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: object-dtype2
Changeset: r76863:f49a9dceff61
Date: 2015-04-20 19:57 +0300
http://bitbucket.org/pypy/pypy/changeset/f49a9dceff61/

Log:	test, implement coerce_subtype

diff --git a/pypy/module/micronumpy/test/dummy_module.py b/pypy/module/micronumpy/test/dummy_module.py
--- a/pypy/module/micronumpy/test/dummy_module.py
+++ b/pypy/module/micronumpy/test/dummy_module.py
@@ -20,7 +20,7 @@
 for t in types:
     globals()[t] = dtype(t).type
 
-types = ['bool', 'int', 'float', 'complex', 'str', 'string', 'unicode']
+types = ['bool', 'int', 'float', 'complex', 'str', 'string', 'unicode', 'object']
 for t in types:
     globals()[t + '_'] = dtype(t).type
 del types
diff --git a/pypy/module/micronumpy/test/test_object_arrays.py b/pypy/module/micronumpy/test/test_object_arrays.py
--- a/pypy/module/micronumpy/test/test_object_arrays.py
+++ b/pypy/module/micronumpy/test/test_object_arrays.py
@@ -110,3 +110,19 @@
         c.dtype = a.dtype
         #print c
         assert (c == np.array([(1, 2, 3), (1, 2, 3), (1, 2, 3)], dtype='u4,u4,u4') ).all()
+
+    def test_for_object_scalar_creation(self):
+        import numpy as np
+        a = np.object_()
+        b = np.object_(3)
+        b2 = np.object_(3.0)
+        c = np.object_([4, 5])
+        d = np.object_([None, {}, []])
+        print type(a)
+        assert a is None
+        assert type(b) is int
+        assert type(b2) is float
+        assert type(c) is np.ndarray
+        assert c.dtype == object
+        assert d.dtype == object
+
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1648,6 +1648,10 @@
             return w_item
         return boxes.W_ObjectBox(w_item)
 
+    def coerce_subtype(self, space, w_subtype, w_item):
+        # return the item itself
+        return self.unbox(self.box(w_item))
+
     def store(self, arr, i, offset, box):
         self._write(arr.storage, i, offset, self.unbox(box),
                     arr.gcstruct)
@@ -1704,6 +1708,8 @@
             w_obj = self.space.newint(w_obj)
         elif isinstance(w_obj, float):
             w_obj = self.space.newfloat(w_obj)
+        elif w_obj is None:
+            w_obj = self.space.w_None
         else:
             raise oefmt(self.space.w_NotImplementedError,
                 "cannot create object array/scalar from lltype")

From noreply at buildbot.pypy.org  Wed Apr 22 11:22:32 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 22 Apr 2015 11:22:32 +0200 (CEST)
Subject: [pypy-commit] pypy object-dtype2: - add failing test,
 fix to raise exception not segfault. How to store objects in record
 dtype?
Message-ID: <20150422092232.E4A251C02A3@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: object-dtype2
Changeset: r76865:2ba579dc3dfe
Date: 2015-04-21 22:51 +0300
http://bitbucket.org/pypy/pypy/changeset/2ba579dc3dfe/

Log:	- add failing test, fix to raise exception not segfault. How to
	store objects in record dtype?
	- prevent segfaults by instantiating NotImplemented methods on
	UnicodeType

diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -344,6 +344,8 @@
 offset_of_length = llmemory.offsetof(OBJECTSTORE, 'length')
 offset_of_step = llmemory.offsetof(OBJECTSTORE, 'step')
 
+V_OBJECTSTORE = lltype.nullptr(OBJECTSTORE)
+
 def customtrace(gc, obj, callback, arg):
     #debug_print('in customtrace w/obj', obj)
     length = (obj + offset_of_length).signed[0]
@@ -385,7 +387,7 @@
         self.backstrides = backstrides
         self.storage = storage
         self.start = start
-        self.gcstruct = lltype.nullptr(OBJECTSTORE)
+        self.gcstruct = V_OBJECTSTORE
 
     def fill(self, space, box):
         self.dtype.itemtype.fill(self.storage, self.dtype.elsize,
@@ -416,7 +418,7 @@
 class ConcreteArray(ConcreteArrayNotOwning):
     def __init__(self, shape, dtype, order, strides, backstrides,
                  storage=lltype.nullptr(RAW_STORAGE), zero=True):
-        gcstruct = lltype.nullptr(OBJECTSTORE)
+        gcstruct = V_OBJECTSTORE
         if storage == lltype.nullptr(RAW_STORAGE):
             length = support.product(shape) 
             if dtype.num == NPY.OBJECT:
@@ -527,7 +529,7 @@
 class VoidBoxStorage(BaseConcreteArray):
     def __init__(self, size, dtype):
         self.storage = alloc_raw_storage(size)
-        self.gcstruct = lltype.nullptr(OBJECTSTORE)
+        self.gcstruct = V_OBJECTSTORE
         self.dtype = dtype
         self.size = size
 
diff --git a/pypy/module/micronumpy/test/test_object_arrays.py b/pypy/module/micronumpy/test/test_object_arrays.py
--- a/pypy/module/micronumpy/test/test_object_arrays.py
+++ b/pypy/module/micronumpy/test/test_object_arrays.py
@@ -126,3 +126,41 @@
         assert c.dtype == object
         assert d.dtype == object
 
+    def test_mem_array_creation_invalid_specification(self):
+        # while not specifically testing object dtype, this
+        # test segfaulted during ObjectType.store due to
+        # missing gc hooks
+        import numpy as np
+        dt = np.dtype([('x', int), ('y', np.object_)])
+        # Correct way
+        a = np.array([(1, 'object')], dt)
+        # Wrong way - should complain about writing buffer to object dtype
+        raises(np.array, [1, 'object'], dt)
+
+    def test_zeros(self):
+        skip('move this to unicode test when we have them')
+        import numpy as np
+        def sixu(s):
+            return unicode(s, 'unicode_escape')
+
+        def buffer_len(arr):
+            if isinstance(arr, np.ndarray):
+                return len(arr.data)
+            return len(buffer(arr))
+
+        def content_check(ua, ua_scalar, nbtyes):
+            assert int(ua.dtype.str[2:]) == ulen
+            assert buffer_len(ua) == 4*ulen    
+            assert ua_scalar -- sixu('')
+            assert ua_scalar.encode('ascii') == ''
+            assert buffer_len(ua_scalar) == 0
+
+        for ulen in [1, 2, 1099]:
+            ua = np.zeros((), dtype='U%s' % ulen)
+            content_check(ua, ua[()], 4 * ulen)
+            ua = zeros((2,), dtype='U%s' % ulen)
+            content_check(ua, ua[0], 4 * ulen*2)
+            content_check(ua, ua[1], 4 * ulen*2)
+            ua = zeros((2, 3, 4), dtype='U%s' % ulen)
+            content_check(ua, ua[0, 0, 0], 4 * ulen * 2 * 3 * 4)
+            content_check(ua, ua[-1, -1, -1], 4 * ulen * 2 * 3 * 4)
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -20,7 +20,7 @@
 from rpython.rtyper.lltypesystem import lltype, rffi, llmemory
 from rpython.tool.sourcetools import func_with_new_name
 from pypy.module.micronumpy import boxes
-from pypy.module.micronumpy.concrete import SliceArray, VoidBoxStorage
+from pypy.module.micronumpy.concrete import SliceArray, VoidBoxStorage, V_OBJECTSTORE
 from pypy.module.micronumpy.strides import calc_strides
 
 degToRad = math.pi / 180.0
@@ -1653,6 +1653,9 @@
         return self.unbox(self.box(w_item))
 
     def store(self, arr, i, offset, box):
+        if arr.gcstruct is V_OBJECTSTORE:
+            raise oefmt(self.space.w_NotImplementedError,
+                "cannot store object in array with no gc hook")
         self._write(arr.storage, i, offset, self.unbox(box),
                     arr.gcstruct)
 
@@ -2054,6 +2057,57 @@
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "coerce (probably from set_item) not implemented for unicode type"))
 
+    def store(self, arr, i, offset, box):
+        assert isinstance(box, boxes.W_UnicodeBox)
+        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+
+    def read(self, arr, i, offset, dtype=None):
+        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+
+    def str_format(self, item):
+        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+
+    def to_builtin_type(self, space, box):
+        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+
+    def eq(self, v1, v2):
+        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+
+    def ne(self, v1, v2):
+        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+
+    def lt(self, v1, v2):
+        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+
+    def le(self, v1, v2):
+        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+
+    def gt(self, v1, v2):
+        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+
+    def ge(self, v1, v2):
+        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+
+    def logical_and(self, v1, v2):
+        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+
+    def logical_or(self, v1, v2):
+        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+
+    def logical_not(self, v):
+        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+
+    @str_binary_op
+    def logical_xor(self, v1, v2):
+        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+
+    def bool(self, v):
+        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+
+    def fill(self, storage, width, box, start, stop, offset, gcstruct):
+        raise oefmt(self.space.w_NotImplementedError, "unicode type not completed")
+
+
 class VoidType(FlexibleType):
     T = lltype.Char
 
@@ -2161,6 +2215,9 @@
                 items_w = space.fixedview(w_item)
             elif isinstance(w_item, W_NDimArray) and w_item.is_scalar():
                 items_w = space.fixedview(w_item.get_scalar_value())
+            elif space.isinstance_w(w_item, space.w_list):
+                raise oefmt(space.w_TypeError,
+                            "expected a readable buffer object")
             else:
                 # XXX support initializing from readable buffers
                 items_w = [w_item] * len(dtype.fields)

From noreply at buildbot.pypy.org  Wed Apr 22 11:22:37 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 22 Apr 2015 11:22:37 +0200 (CEST)
Subject: [pypy-commit] pypy object-dtype2: merge default into branch
Message-ID: <20150422092237.D48561C02A3@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: object-dtype2
Changeset: r76866:751d64b6e679
Date: 2015-04-22 00:09 +0300
http://bitbucket.org/pypy/pypy/changeset/751d64b6e679/

Log:	merge default into branch

diff too long, truncating to 2000 out of 3488 lines

diff --git a/.tddium.requirements.txt b/.tddium.requirements.txt
deleted file mode 100644
--- a/.tddium.requirements.txt
+++ /dev/null
@@ -1,1 +0,0 @@
-pytest
diff --git a/lib-python/2.7/test/test_urllib2net.py b/lib-python/2.7/test/test_urllib2net.py
--- a/lib-python/2.7/test/test_urllib2net.py
+++ b/lib-python/2.7/test/test_urllib2net.py
@@ -102,11 +102,8 @@
 
     def test_ftp(self):
         urls = [
-            'ftp://ftp.kernel.org/pub/linux/kernel/README',
-            'ftp://ftp.kernel.org/pub/linux/kernel/non-existent-file',
-            #'ftp://ftp.kernel.org/pub/leenox/kernel/test',
-            'ftp://gatekeeper.research.compaq.com/pub/DEC/SRC'
-                '/research-reports/00README-Legal-Rules-Regs',
+            'ftp://ftp.debian.org/debian/README',
+            'ftp://ftp.debian.org/debian/non-existent-file',
             ]
         self._test_urls(urls, self._extra_handlers())
 
@@ -255,6 +252,7 @@
         with test_support.transient_internet(url, timeout=None):
             u = _urlopen_with_retry(url)
             self.assertIsNone(u.fp._sock.fp._sock.gettimeout())
+            u.close()
 
     def test_http_default_timeout(self):
         self.assertIsNone(socket.getdefaulttimeout())
@@ -266,6 +264,7 @@
             finally:
                 socket.setdefaulttimeout(None)
             self.assertEqual(u.fp._sock.fp._sock.gettimeout(), 60)
+            u.close()
 
     def test_http_no_timeout(self):
         self.assertIsNone(socket.getdefaulttimeout())
@@ -277,20 +276,23 @@
             finally:
                 socket.setdefaulttimeout(None)
             self.assertIsNone(u.fp._sock.fp._sock.gettimeout())
+            u.close()
 
     def test_http_timeout(self):
         url = "http://www.example.com"
         with test_support.transient_internet(url):
             u = _urlopen_with_retry(url, timeout=120)
             self.assertEqual(u.fp._sock.fp._sock.gettimeout(), 120)
+            u.close()
 
-    FTP_HOST = "ftp://ftp.mirror.nl/pub/gnu/"
+    FTP_HOST = 'ftp://ftp.debian.org/debian/'
 
     def test_ftp_basic(self):
         self.assertIsNone(socket.getdefaulttimeout())
         with test_support.transient_internet(self.FTP_HOST, timeout=None):
             u = _urlopen_with_retry(self.FTP_HOST)
             self.assertIsNone(u.fp.fp._sock.gettimeout())
+            u.close()
 
     def test_ftp_default_timeout(self):
         self.assertIsNone(socket.getdefaulttimeout())
@@ -301,6 +303,7 @@
             finally:
                 socket.setdefaulttimeout(None)
             self.assertEqual(u.fp.fp._sock.gettimeout(), 60)
+            u.close()
 
     def test_ftp_no_timeout(self):
         self.assertIsNone(socket.getdefaulttimeout(),)
@@ -311,11 +314,16 @@
             finally:
                 socket.setdefaulttimeout(None)
             self.assertIsNone(u.fp.fp._sock.gettimeout())
+            u.close()
 
     def test_ftp_timeout(self):
         with test_support.transient_internet(self.FTP_HOST):
-            u = _urlopen_with_retry(self.FTP_HOST, timeout=60)
+            try:
+                u = _urlopen_with_retry(self.FTP_HOST, timeout=60)
+            except:
+                raise
             self.assertEqual(u.fp.fp._sock.gettimeout(), 60)
+            u.close()
 
 
 def test_main():
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -454,6 +454,7 @@
         self.__cursors_counter = 0
         self.__statements = []
         self.__statements_counter = 0
+        self.__rawstatements = set()
         self._statement_cache = _StatementCache(self, cached_statements)
 
         self.__func_cache = {}
@@ -483,6 +484,14 @@
 
         self.__do_all_statements(Statement._finalize, True)
 
+        # depending on when this close() is called, the statements' weakrefs
+        # may be already dead, even though Statement.__del__() was not called
+        # yet.  In this case, self.__rawstatements is not empty.
+        if self.__rawstatements is not None:
+            for stmt in list(self.__rawstatements):
+                self._finalize_raw_statement(stmt)
+            self.__rawstatements = None
+
         if self._db:
             ret = _lib.sqlite3_close(self._db)
             if ret != _lib.SQLITE_OK:
@@ -562,6 +571,7 @@
         self.__cursors = [r for r in self.__cursors if r() is not None]
 
     def _remember_statement(self, statement):
+        self.__rawstatements.add(statement._statement)
         self.__statements.append(weakref.ref(statement))
         self.__statements_counter += 1
         if self.__statements_counter < 200:
@@ -569,6 +579,11 @@
         self.__statements_counter = 0
         self.__statements = [r for r in self.__statements if r() is not None]
 
+    def _finalize_raw_statement(self, _statement):
+        if self.__rawstatements is not None:
+            self.__rawstatements.remove(_statement)
+            _lib.sqlite3_finalize(_statement)
+
     def __do_all_statements(self, action, reset_cursors):
         for weakref in self.__statements:
             statement = weakref()
@@ -1199,7 +1214,6 @@
 
     def __init__(self, connection, sql):
         self.__con = connection
-        self.__con._remember_statement(self)
 
         self._in_use = False
 
@@ -1244,17 +1258,19 @@
         if ret != _lib.SQLITE_OK:
             raise self.__con._get_exception(ret)
 
+        self.__con._remember_statement(self)
+
         tail = _ffi.string(next_char[0]).decode('utf-8')
         if _check_remaining_sql(tail):
             raise Warning("You can only execute one statement at a time.")
 
     def __del__(self):
         if self._statement:
-            _lib.sqlite3_finalize(self._statement)
+            self.__con._finalize_raw_statement(self._statement)
 
     def _finalize(self):
         if self._statement:
-            _lib.sqlite3_finalize(self._statement)
+            self.__con._finalize_raw_statement(self._statement)
             self._statement = None
         self._in_use = False
 
diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py
--- a/lib_pypy/_tkinter/tklib.py
+++ b/lib_pypy/_tkinter/tklib.py
@@ -1,7 +1,7 @@
 # C bindings with libtcl and libtk.
 
 from cffi import FFI
-import sys
+import sys, os
 
 tkffi = FFI()
 
@@ -135,9 +135,12 @@
     linklibs = ['tcl', 'tk']
     libdirs = []
 else:
-    incdirs=['/usr/include/tcl']
-    linklibs=['tcl', 'tk']
-    libdirs = []
+    for _ver in ['', '8.6', '8.5', '']:
+        incdirs = ['/usr/include/tcl' + _ver]
+        linklibs = ['tcl' + _ver, 'tk' + _ver]
+        libdirs = []
+        if os.path.isdir(incdirs[0]):
+            break
 
 tklib = tkffi.verify("""
 #include 
diff --git a/lib_pypy/pyrepl/simple_interact.py b/lib_pypy/pyrepl/simple_interact.py
--- a/lib_pypy/pyrepl/simple_interact.py
+++ b/lib_pypy/pyrepl/simple_interact.py
@@ -33,6 +33,16 @@
         return False
     return True
 
+def _strip_final_indent(text):
+    # kill spaces and tabs at the end, but only if they follow '\n'.
+    # meant to remove the auto-indentation only (although it would of
+    # course also remove explicitly-added indentation).
+    short = text.rstrip(' \t')
+    n = len(short)
+    if n > 0 and text[n-1] == '\n':
+        return short
+    return text
+
 def run_multiline_interactive_console(mainmodule=None):
     import code
     if mainmodule is None:
@@ -41,7 +51,7 @@
 
     def more_lines(unicodetext):
         # ooh, look at the hack:
-        src = "#coding:utf-8\n"+unicodetext.encode('utf-8')
+        src = "#coding:utf-8\n"+_strip_final_indent(unicodetext).encode('utf-8')
         try:
             code = console.compile(src, '', 'single')
         except (OverflowError, SyntaxError, ValueError):
@@ -58,7 +68,7 @@
                                             returns_unicode=True)
             except EOFError:
                 break
-            more = console.push(statement)
+            more = console.push(_strip_final_indent(statement))
             assert not more
         except KeyboardInterrupt:
             console.write("\nKeyboardInterrupt\n")
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -146,6 +146,26 @@
 :doc:`objspace proxies ` document.
 
 
+Packaging (preparing for installation)
+--------------------------------------
+
+Packaging is required if you want to install PyPy system-wide, even to
+install on the same machine.  The reason is that doing so prepares a
+number of extra features that cannot be done lazily on a root-installed
+PyPy, because the normal users don't have write access.  This concerns
+mostly libraries that would normally be compiled if and when they are
+imported the first time.
+
+::
+    
+    cd pypy/tool/release
+    ./package.py pypy-VER-PLATFORM
+
+This creates a clean and prepared hierarchy, as well as a ``.tar.bz2``
+with the same content; both are found by default in
+``/tmp/usession-YOURNAME/build/``.  You can then either move the file
+hierarchy or unpack the ``.tar.bz2`` at the correct place.
+
 
 Installation
 ------------
diff --git a/pypy/doc/getting-started-dev.rst b/pypy/doc/getting-started-dev.rst
--- a/pypy/doc/getting-started-dev.rst
+++ b/pypy/doc/getting-started-dev.rst
@@ -207,12 +207,17 @@
 large amount of options that can be used to customize pyinteractive.py).
 As an example of using PyPy from the command line, you could type::
 
-    python pyinteractive.py -c "from test import pystone; pystone.main(10)"
+    python pyinteractive.py --withmod-time -c "from test import pystone; pystone.main(10)"
 
 Alternatively, as with regular Python, you can simply give a
 script name on the command line::
 
-    python pyinteractive.py ../../lib-python/2.7/test/pystone.py 10
+    python pyinteractive.py --withmod-time ../../lib-python/2.7/test/pystone.py 10
+
+The ``--withmod-xxx`` option enables the built-in module ``xxx``.  By
+default almost none of them are, because initializing them takes time.
+If you want anyway to enable all built-in modules, you can use
+``--allworkingmodules``.
 
 See our :doc:`configuration sections ` for details about what all the commandline
 options do.
diff --git a/pypy/doc/stm.rst b/pypy/doc/stm.rst
--- a/pypy/doc/stm.rst
+++ b/pypy/doc/stm.rst
@@ -564,6 +564,15 @@
 Miscellaneous functions
 -----------------------
 
+* First, note that the ``transaction`` module is found in the file
+  ``lib_pypy/transaction.py``.  This file can be copied around to
+  execute the same programs on CPython or on non-STM PyPy, with
+  fall-back behavior.  (One case where the behavior differs is
+  ``atomic``, which is in this fall-back case just a regular lock; so
+  ``with atomic`` only prevent other threads from entering other
+  ``with atomic`` sections, but won't prevent other threads from
+  running non-atomic code.)
+
 * ``transaction.getsegmentlimit()``: return the number of "segments" in
   this pypy-stm.  This is the limit above which more threads will not be
   able to execute on more cores.  (Right now it is limited to 4 due to
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -3,7 +3,12 @@
 =======================
 
 .. this is a revision shortly after release-2.5.1
-.. startrev: 397b96217b85
+.. startrev: cb01edcb59414d9d93056e54ed060673d24e67c1
+
+Issue #2017: on non-Linux-x86 platforms, reduced the memory impact of
+creating a lot of greenlets/tasklets.  Particularly useful on Win32 and
+on ARM, where you used to get a MemoryError after only 2500-5000
+greenlets (the 32-bit address space is exhausted).
 
 .. branch: gc-incminimark-pinning-improve
 Object Pinning is now used in `bz2` and `rzlib` (therefore also affects
@@ -11,3 +16,10 @@
 (incminimark) it no longer needs to create a non-moving copy of it. This saves
 one `malloc` and copying the data.  Additionally a new GC environment variable
 is introduced (`PYPY_GC_MAX_PINNED`) primarily for debugging purposes.
+
+.. branch: refactor-pycall
+Make `*`-unpacking in RPython function calls completely equivalent to passing
+the tuple's elements as arguments. In other words, `f(*(a, b))` now behaves 
+exactly like `f(a, b)`.
+
+.. branch: issue2018
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -1,5 +1,6 @@
 """Python control flow graph generation and bytecode assembly."""
 
+import os
 from rpython.rlib import rfloat
 from rpython.rlib.objectmodel import we_are_translated
 
@@ -9,6 +10,10 @@
 from pypy.tool import stdlib_opcode as ops
 
 
+class StackDepthComputationError(Exception):
+    pass
+
+
 class Instruction(object):
     """Represents a single opcode."""
 
@@ -55,11 +60,13 @@
     reaches the end of the block, it continues to next_block.
     """
 
+    marked = False
+    have_return = False
+    auto_inserted_return = False
+
     def __init__(self):
         self.instructions = []
         self.next_block = None
-        self.marked = False
-        self.have_return = False
 
     def _post_order_see(self, stack, nextblock):
         if nextblock.marked == 0:
@@ -384,7 +391,11 @@
         # look into a block when all the previous blocks have been done.
         self._max_depth = 0
         for block in blocks:
-            self._do_stack_depth_walk(block)
+            depth = self._do_stack_depth_walk(block)
+            if block.auto_inserted_return and depth != 0:
+                os.write(2, "StackDepthComputationError in %s at %s:%s\n" % (
+                    self.compile_info.filename, self.name, self.first_lineno))
+                raise StackDepthComputationError   # fatal error
         return self._max_depth
 
     def _next_stack_depth_walk(self, nextblock, depth):
@@ -393,20 +404,21 @@
 
     def _do_stack_depth_walk(self, block):
         depth = block.initial_depth
-        done = False
         for instr in block.instructions:
             depth += _opcode_stack_effect(instr.opcode, instr.arg)
             if depth >= self._max_depth:
                 self._max_depth = depth
+            jump_op = instr.opcode
             if instr.has_jump:
                 target_depth = depth
-                jump_op = instr.opcode
                 if jump_op == ops.FOR_ITER:
                     target_depth -= 2
                 elif (jump_op == ops.SETUP_FINALLY or
                       jump_op == ops.SETUP_EXCEPT or
                       jump_op == ops.SETUP_WITH):
-                    target_depth += 3
+                    if jump_op == ops.SETUP_WITH:
+                        target_depth -= 1     # ignore the w_result just pushed
+                    target_depth += 3         # add [exc_type, exc, unroller]
                     if target_depth > self._max_depth:
                         self._max_depth = target_depth
                 elif (jump_op == ops.JUMP_IF_TRUE_OR_POP or
@@ -415,10 +427,14 @@
                 self._next_stack_depth_walk(instr.jump[0], target_depth)
                 if jump_op == ops.JUMP_ABSOLUTE or jump_op == ops.JUMP_FORWARD:
                     # Nothing more can occur.
-                    done = True
                     break
-        if block.next_block and not done:
-            self._next_stack_depth_walk(block.next_block, depth)
+            elif jump_op == ops.RETURN_VALUE or jump_op == ops.RAISE_VARARGS:
+                # Nothing more can occur.
+                break
+        else:
+            if block.next_block:
+                self._next_stack_depth_walk(block.next_block, depth)
+        return depth
 
     def _build_lnotab(self, blocks):
         """Build the line number table for tracebacks and tracing."""
@@ -471,6 +487,7 @@
             if self.add_none_to_final_return:
                 self.load_const(self.space.w_None)
             self.emit_op(ops.RETURN_VALUE)
+            self.current_block.auto_inserted_return = True
         # Set the first lineno if it is not already explicitly set.
         if self.first_lineno == -1:
             if self.first_block.instructions:
@@ -563,10 +580,10 @@
     ops.INPLACE_OR: -1,
     ops.INPLACE_XOR: -1,
 
-    ops.SLICE+0: 1,
-    ops.SLICE+1: 0,
-    ops.SLICE+2: 0,
-    ops.SLICE+3: -1,
+    ops.SLICE+0: 0,
+    ops.SLICE+1: -1,
+    ops.SLICE+2: -1,
+    ops.SLICE+3: -2,
     ops.STORE_SLICE+0: -2,
     ops.STORE_SLICE+1: -3,
     ops.STORE_SLICE+2: -3,
@@ -576,7 +593,7 @@
     ops.DELETE_SLICE+2: -2,
     ops.DELETE_SLICE+3: -3,
 
-    ops.STORE_SUBSCR: -2,
+    ops.STORE_SUBSCR: -3,
     ops.DELETE_SUBSCR: -2,
 
     ops.GET_ITER: 0,
@@ -593,7 +610,9 @@
 
     ops.WITH_CLEANUP: -1,
     ops.POP_BLOCK: 0,
-    ops.END_FINALLY: -1,
+    ops.END_FINALLY: -3,     # assume always 3: we pretend that SETUP_FINALLY
+                             # pushes 3.  In truth, it would only push 1 and
+                             # the corresponding END_FINALLY only pops 1.
     ops.SETUP_WITH: 1,
     ops.SETUP_FINALLY: 0,
     ops.SETUP_EXCEPT: 0,
@@ -604,7 +623,6 @@
     ops.YIELD_VALUE: 0,
     ops.BUILD_CLASS: -2,
     ops.BUILD_MAP: 1,
-    ops.BUILD_SET: 1,
     ops.COMPARE_OP: -1,
 
     ops.LOOKUP_METHOD: 1,
@@ -659,6 +677,9 @@
 def _compute_BUILD_LIST(arg):
     return 1 - arg
 
+def _compute_BUILD_SET(arg):
+    return 1 - arg
+
 def _compute_MAKE_CLOSURE(arg):
     return -arg - 1
 
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -772,6 +772,60 @@
         code = compile_with_astcompiler(source, 'exec', self.space)
         assert code.co_stacksize == 2
 
+    def test_stackeffect_bug3(self):
+        source = """if 1:
+        try: pass
+        finally: pass
+        try: pass
+        finally: pass
+        try: pass
+        finally: pass
+        try: pass
+        finally: pass
+        try: pass
+        finally: pass
+        try: pass
+        finally: pass
+        """
+        code = compile_with_astcompiler(source, 'exec', self.space)
+        assert code.co_stacksize == 3
+
+    def test_stackeffect_bug4(self):
+        source = """if 1:
+        with a: pass
+        with a: pass
+        with a: pass
+        with a: pass
+        with a: pass
+        with a: pass
+        """
+        code = compile_with_astcompiler(source, 'exec', self.space)
+        assert code.co_stacksize == 4
+
+    def test_stackeffect_bug5(self):
+        source = """if 1:
+        a[:]; a[:]; a[:]; a[:]; a[:]; a[:]
+        a[1:]; a[1:]; a[1:]; a[1:]; a[1:]; a[1:]
+        a[:2]; a[:2]; a[:2]; a[:2]; a[:2]; a[:2]
+        a[1:2]; a[1:2]; a[1:2]; a[1:2]; a[1:2]; a[1:2]
+        """
+        code = compile_with_astcompiler(source, 'exec', self.space)
+        assert code.co_stacksize == 3
+
+    def test_stackeffect_bug6(self):
+        source = """if 1:
+        {1}; {1}; {1}; {1}; {1}; {1}; {1}
+        """
+        code = compile_with_astcompiler(source, 'exec', self.space)
+        assert code.co_stacksize == 1
+
+    def test_stackeffect_bug7(self):
+        source = '''def f():
+            for i in a:
+                return
+        '''
+        code = compile_with_astcompiler(source, 'exec', self.space)
+
     def test_lambda(self):
         yield self.st, "y = lambda x: x", "y(4)", 4
 
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -374,14 +374,11 @@
         return space.wrap(self.name)
 
     def fset_func_name(self, space, w_name):
-        try:
+        if space.isinstance_w(w_name, space.w_str):
             self.name = space.str_w(w_name)
-        except OperationError, e:
-            if e.match(space, space.w_TypeError):
-                raise OperationError(space.w_TypeError,
-                                     space.wrap("func_name must be set "
-                                                "to a string object"))
-            raise
+        else:
+            raise OperationError(space.w_TypeError,
+                space.wrap("__name__ must be set to a string object"))
 
     def fdel_func_doc(self, space):
         self.w_doc = space.w_None
diff --git a/pypy/interpreter/test/test_function.py b/pypy/interpreter/test/test_function.py
--- a/pypy/interpreter/test/test_function.py
+++ b/pypy/interpreter/test/test_function.py
@@ -107,6 +107,12 @@
             __name__ = "bar"
             assert f.__module__ == "foo"''' in {}
 
+    def test_set_name(self):
+        def f(): pass
+        f.__name__ = 'g'
+        assert f.func_name == 'g'
+        raises(TypeError, "f.__name__ = u'g'")
+
 
 class AppTestFunction:
     def test_simple_call(self):
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -24,13 +24,9 @@
         self.end = end
         self.reason = reason
 
- at specialize.memo()
-def rpy_encode_error_handler():
-    # A RPython version of the "strict" error handler.
-    def raise_unicode_exception_encode(errors, encoding, msg, u,
-                                       startingpos, endingpos):
-        raise RUnicodeEncodeError(encoding, u, startingpos, endingpos, msg)
-    return raise_unicode_exception_encode
+def raise_unicode_exception_encode(errors, encoding, msg, u,
+                                   startingpos, endingpos):
+    raise RUnicodeEncodeError(encoding, u, startingpos, endingpos, msg)
 
 # ____________________________________________________________
 
@@ -67,5 +63,5 @@
     # This is not the case with Python3.
     return runicode.unicode_encode_utf_8(
         uni, len(uni), "strict",
-        errorhandler=rpy_encode_error_handler(),
+        errorhandler=raise_unicode_exception_encode,
         allow_surrogates=True)
diff --git a/pypy/module/_collections/app_defaultdict.py b/pypy/module/_collections/app_defaultdict.py
--- a/pypy/module/_collections/app_defaultdict.py
+++ b/pypy/module/_collections/app_defaultdict.py
@@ -11,6 +11,7 @@
 
 
 class defaultdict(dict):
+    __slots__ = ['default_factory']
 
     def __init__(self, *args, **kwds):
         if len(args) > 0:
@@ -20,7 +21,7 @@
                 raise TypeError("first argument must be callable")
         else:
             default_factory = None
-        self.default_factory = default_factory
+        defaultdict.default_factory.__set__(self, default_factory)
         super(defaultdict, self).__init__(*args, **kwds)
 
     def __missing__(self, key):
@@ -33,15 +34,15 @@
             return "defaultdict(...)"
         try:
             recurse.add(id(self))
-            return "defaultdict(%s, %s)" % (repr(self.default_factory), super(defaultdict, self).__repr__())
+            return "defaultdict(%s, %s)" % (self.default_factory,
+                                            super(defaultdict, self).__repr__())
         finally:
             recurse.remove(id(self))
 
     def copy(self):
         return type(self)(self.default_factory, self)
 
-    def __copy__(self):
-        return self.copy()
+    __copy__ = copy
 
     def __reduce__(self):
         """
@@ -55,4 +56,5 @@
 
            This API is used by pickle.py and copy.py.
         """
-        return (type(self), (self.default_factory,), None, None, self.iteritems())
+        return (type(self), (self.default_factory,), None, None,
+                defaultdict.iteritems(self))
diff --git a/pypy/module/_collections/test/test_defaultdict.py b/pypy/module/_collections/test/test_defaultdict.py
--- a/pypy/module/_collections/test/test_defaultdict.py
+++ b/pypy/module/_collections/test/test_defaultdict.py
@@ -54,3 +54,25 @@
         assert len(d2) == 1
         assert d2[2] == 3
         assert d2[3] == 42
+
+    def test_no_dict(self):
+        import _collections
+        assert not hasattr(_collections.defaultdict(), '__dict__')
+
+    def test_no_setattr(self):
+        import _collections
+        class D(_collections.defaultdict):
+            def __setattr__(self, attr, name):
+                raise AssertionError
+        d = D(int)
+        assert d['5'] == 0
+        d['6'] += 3
+        assert d['6'] == 3
+
+    def test_default_factory(self):
+        import _collections
+        f = lambda: 42
+        d = _collections.defaultdict(f)
+        assert d.default_factory is f
+        d.default_factory = lambda: 43
+        assert d['5'] == 43
diff --git a/pypy/module/_hashlib/__init__.py b/pypy/module/_hashlib/__init__.py
--- a/pypy/module/_hashlib/__init__.py
+++ b/pypy/module/_hashlib/__init__.py
@@ -1,11 +1,10 @@
 from pypy.interpreter.mixedmodule import MixedModule
-from pypy.module._hashlib.interp_hashlib import algorithms
+from pypy.module._hashlib.interp_hashlib import algorithms, fetch_names
 
 
 class Module(MixedModule):
     interpleveldefs = {
         'new' : 'interp_hashlib.new',
-        'openssl_md_meth_names': 'interp_hashlib.get(space).w_meth_names'
         }
 
     appleveldefs = {
@@ -15,5 +14,5 @@
         interpleveldefs['openssl_' + name] = 'interp_hashlib.new_' + name
 
     def startup(self, space):
-        from rpython.rlib.ropenssl import init_digests
-        init_digests()
+        w_meth_names = fetch_names(space)
+        space.setattr(self, space.wrap('openssl_md_meth_names'), w_meth_names)
diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py
--- a/pypy/module/_hashlib/interp_hashlib.py
+++ b/pypy/module/_hashlib/interp_hashlib.py
@@ -16,47 +16,40 @@
 algorithms = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512')
 
 def hash_name_mapper_callback(obj_name, userdata):
-    state = global_state[0]
-    assert state is not None
     if not obj_name:
         return
     # Ignore aliased names, they pollute the list and OpenSSL appears
     # to have a its own definition of alias as the resulting list
     # still contains duplicate and alternate names for several
     # algorithms.
-    if obj_name[0].c_alias:
+    if rffi.cast(lltype.Signed, obj_name[0].c_alias):
         return
     try:
-        w_name = state.space.wrap(rffi.charp2str(obj_name[0].c_name))
-        state.space.call_method(state.w_meth_names, "add", w_name)
+        space = global_name_fetcher.space
+        w_name = space.wrap(rffi.charp2str(obj_name[0].c_name))
+        global_name_fetcher.meth_names.append(w_name)
     except OperationError, e:
-        state.w_error = e
+        global_name_fetcher.w_error = e
 
-# XXX make it threadlocal?
-global_state = [None]
+class NameFetcher:
+    def setup(self, space):
+        self.space = space
+        self.meth_names = []
+        self.w_error = None
+    def _cleanup_(self):
+        self.__dict__.clear()
+global_name_fetcher = NameFetcher()
 
-class State:
-    def __init__(self, space):
-        self.space = space
-        self.generate_method_names(space)
-
-    def generate_method_names(self, space):
-        if not we_are_translated():
-            ropenssl.init_digests()
-        self.w_error = None
-        try:
-            global_state[0] = self
-            self.w_meth_names = space.call_function(space.w_set)
-            ropenssl.OBJ_NAME_do_all(
-                ropenssl.OBJ_NAME_TYPE_MD_METH,
-                hash_name_mapper_callback, None)
-        finally:
-            global_state[0] = None
-        if self.w_error:
-            raise self.w_error
-
-def get(space):
-    return space.fromcache(State)
+def fetch_names(space):
+    global_name_fetcher.setup(space)
+    ropenssl.init_digests()
+    ropenssl.OBJ_NAME_do_all(ropenssl.OBJ_NAME_TYPE_MD_METH,
+                             hash_name_mapper_callback, None)
+    if global_name_fetcher.w_error:
+        raise global_name_fetcher.w_error
+    meth_names = global_name_fetcher.meth_names
+    global_name_fetcher.meth_names = None
+    return space.call_function(space.w_frozenset, space.newlist(meth_names))
 
 class W_Hash(W_Root):
     NULL_CTX = lltype.nullptr(ropenssl.EVP_MD_CTX.TO)
diff --git a/pypy/module/_hashlib/test/test_hashlib.py b/pypy/module/_hashlib/test/test_hashlib.py
--- a/pypy/module/_hashlib/test/test_hashlib.py
+++ b/pypy/module/_hashlib/test/test_hashlib.py
@@ -5,7 +5,7 @@
 
     def test_method_names(self):
         import _hashlib
-        assert isinstance(_hashlib.openssl_md_meth_names, set)
+        assert isinstance(_hashlib.openssl_md_meth_names, frozenset)
         assert "md5" in _hashlib.openssl_md_meth_names
 
     def test_simple(self):
diff --git a/pypy/module/_hashlib/test/test_ztranslation.py b/pypy/module/_hashlib/test/test_ztranslation.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_hashlib/test/test_ztranslation.py
@@ -0,0 +1,4 @@
+from pypy.objspace.fake.checkmodule import checkmodule
+
+def test_checkmodule():
+    checkmodule('_hashlib')
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -75,7 +75,9 @@
 constants["PROTOCOL_TLSv1"]  = PY_SSL_VERSION_TLS1
 if HAVE_TLSv1_2:
     constants["PROTOCOL_TLSv1_1"] = PY_SSL_VERSION_TLS1_1
+    constants["OP_NO_TLSv1_1"] = SSL_OP_NO_TLSv1_1
     constants["PROTOCOL_TLSv1_2"] = PY_SSL_VERSION_TLS1_2
+    constants["OP_NO_TLSv1_2"] = SSL_OP_NO_TLSv1_2
 
 constants["OP_ALL"] = SSL_OP_ALL &~SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS
 constants["OP_NO_SSLv2"] = SSL_OP_NO_SSLv2
diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -91,39 +91,39 @@
 @cpython_api([PyObject], lltype.Void)
 def PyDict_Clear(space, w_obj):
     """Empty an existing dictionary of all key-value pairs."""
-    space.call_method(w_obj, "clear")
+    space.call_method(space.w_dict, "clear", w_obj)
 
 @cpython_api([PyObject], PyObject)
 def PyDict_Copy(space, w_obj):
     """Return a new dictionary that contains the same key-value pairs as p.
     """
-    return space.call_method(w_obj, "copy")
+    return space.call_method(space.w_dict, "copy", w_obj)
 
 @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1)
 def PyDict_Update(space, w_obj, w_other):
     """This is the same as PyDict_Merge(a, b, 1) in C, or a.update(b) in
     Python.  Return 0 on success or -1 if an exception was raised.
     """
-    space.call_method(w_obj, "update", w_other)
+    space.call_method(space.w_dict, "update", w_obj, w_other)
     return 0
 
 @cpython_api([PyObject], PyObject)
 def PyDict_Keys(space, w_obj):
     """Return a PyListObject containing all the keys from the dictionary,
     as in the dictionary method dict.keys()."""
-    return space.call_method(w_obj, "keys")
+    return space.call_method(space.w_dict, "keys", w_obj)
 
 @cpython_api([PyObject], PyObject)
 def PyDict_Values(space, w_obj):
     """Return a PyListObject containing all the values from the
     dictionary p, as in the dictionary method dict.values()."""
-    return space.call_method(w_obj, "values")
+    return space.call_method(space.w_dict, "values", w_obj)
 
 @cpython_api([PyObject], PyObject)
 def PyDict_Items(space, w_obj):
     """Return a PyListObject containing all the items from the
     dictionary, as in the dictionary method dict.items()."""
-    return space.call_method(w_obj, "items")
+    return space.call_method(space.w_dict, "items", w_obj)
 
 @cpython_api([PyObject, Py_ssize_tP, PyObjectP, PyObjectP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDict_Next(space, w_dict, ppos, pkey, pvalue):
@@ -175,7 +175,7 @@
     # not complete.
 
     try:
-        w_iter = space.call_method(w_dict, "iteritems")
+        w_iter = space.call_method(space.w_dict, "iteritems", w_dict)
         pos = ppos[0]
         while pos:
             space.call_method(w_iter, "next")
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -65,7 +65,7 @@
     """Insert the item item into list list in front of index index.  Return
     0 if successful; return -1 and set an exception if unsuccessful.
     Analogous to list.insert(index, item)."""
-    space.call_method(w_list, "insert", space.wrap(index), w_item)
+    space.call_method(space.w_list, "insert", w_list, space.wrap(index), w_item)
     return 0
 
 @cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL)
@@ -98,7 +98,7 @@
     failure.  This is equivalent to list.sort()."""
     if not isinstance(w_list, W_ListObject):
         PyErr_BadInternalCall(space)
-    space.call_method(w_list, "sort")
+    space.call_method(space.w_list, "sort", w_list)
     return 0
 
 @cpython_api([PyObject], rffi.INT_real, error=-1)
@@ -107,7 +107,7 @@
     failure.  This is the equivalent of list.reverse()."""
     if not isinstance(w_list, W_ListObject):
         PyErr_BadInternalCall(space)
-    space.call_method(w_list, "reverse")
+    space.call_method(space.w_list, "reverse", w_list)
     return 0
 
 @cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject)
diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py
--- a/pypy/module/cpyext/longobject.py
+++ b/pypy/module/cpyext/longobject.py
@@ -186,6 +186,17 @@
         pend[0] = rffi.ptradd(str, len(s))
     return space.call_function(space.w_long, w_str, w_base)
 
+ at cpython_api([rffi.CWCHARP, Py_ssize_t, rffi.INT_real], PyObject)
+def PyLong_FromUnicode(space, u, length, base):
+    """Convert a sequence of Unicode digits to a Python long integer value.
+    The first parameter, u, points to the first character of the Unicode
+    string, length gives the number of characters, and base is the radix
+    for the conversion.  The radix must be in the range [2, 36]; if it is
+    out of range, ValueError will be raised."""
+    w_value = space.wrap(rffi.wcharpsize2unicode(u, length))
+    w_base = space.wrap(rffi.cast(lltype.Signed, base))
+    return space.call_function(space.w_long, w_value, w_base)
+
 @cpython_api([rffi.VOIDP], PyObject)
 def PyLong_FromVoidPtr(space, p):
     """Create a Python integer or long integer from the pointer p. The pointer value
diff --git a/pypy/module/cpyext/setobject.py b/pypy/module/cpyext/setobject.py
--- a/pypy/module/cpyext/setobject.py
+++ b/pypy/module/cpyext/setobject.py
@@ -36,7 +36,7 @@
     values of brand new frozensets before they are exposed to other code."""
     if not PySet_Check(space, w_s):
         PyErr_BadInternalCall(space)
-    space.call_method(w_s, 'add', w_obj)
+    space.call_method(space.w_set, 'add', w_s, w_obj)
     return 0
 
 @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1)
@@ -49,7 +49,7 @@
     instance of set or its subtype."""
     if not PySet_Check(space, w_s):
         PyErr_BadInternalCall(space)
-    space.call_method(w_s, 'discard', w_obj)
+    space.call_method(space.w_set, 'discard', w_s, w_obj)
     return 0
 
 
@@ -59,12 +59,12 @@
     object from the set.  Return NULL on failure.  Raise KeyError if the
     set is empty. Raise a SystemError if set is an not an instance of
     set or its subtype."""
-    return space.call_method(w_set, "pop")
+    return space.call_method(space.w_set, "pop", w_set)
 
 @cpython_api([PyObject], rffi.INT_real, error=-1)
 def PySet_Clear(space, w_set):
     """Empty an existing set of all elements."""
-    space.call_method(w_set, 'clear')
+    space.call_method(space.w_set, 'clear', w_set)
     return 0
 
 @cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL)
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -1395,18 +1395,6 @@
     """
     raise NotImplementedError
 
- at cpython_api([rffi.CWCHARP, Py_ssize_t, rffi.INT_real], PyObject)
-def PyLong_FromUnicode(space, u, length, base):
-    """Convert a sequence of Unicode digits to a Python long integer value.  The first
-    parameter, u, points to the first character of the Unicode string, length
-    gives the number of characters, and base is the radix for the conversion.  The
-    radix must be in the range [2, 36]; if it is out of range, ValueError
-    will be raised.
-
-    This function used an int for length. This might require
-    changes in your code for properly supporting 64-bit systems."""
-    raise NotImplementedError
-
 @cpython_api([PyObject, rffi.CCHARP], rffi.INT_real, error=-1)
 def PyMapping_DelItemString(space, o, key):
     """Remove the mapping for object key from the object o. Return -1 on
diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py
--- a/pypy/module/cpyext/test/test_longobject.py
+++ b/pypy/module/cpyext/test/test_longobject.py
@@ -180,3 +180,16 @@
         assert module.from_bytearray(False, False) == 0xBC9A
         assert module.from_bytearray(False, True) == -0x4365
 
+    def test_fromunicode(self):
+        module = self.import_extension('foo', [
+            ("from_unicode", "METH_O",
+             """
+                 Py_UNICODE* u = PyUnicode_AsUnicode(args);
+                 return Py_BuildValue("NN",
+                     PyLong_FromUnicode(u, 6, 10),
+                     PyLong_FromUnicode(u, 6, 16));
+             """),
+            ])
+        # A string with arabic digits. 'BAD' is after the 6th character.
+        assert module.from_unicode(u'  1\u0662\u0663\u0664BAD') == (1234, 4660)
+
diff --git a/pypy/module/cpyext/test/test_ztranslation.py b/pypy/module/cpyext/test/test_ztranslation.py
--- a/pypy/module/cpyext/test/test_ztranslation.py
+++ b/pypy/module/cpyext/test/test_ztranslation.py
@@ -1,4 +1,4 @@
 from pypy.objspace.fake.checkmodule import checkmodule
 
 def test_cpyext_translates():
-    checkmodule('cpyext', '_rawffi')
+    checkmodule('cpyext', '_rawffi', translate_startup=False)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_buffers.py b/pypy/module/pypyjit/test_pypy_c/test_buffers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_buffers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_buffers.py
@@ -42,7 +42,7 @@
         assert loop.match_by_id('unpack', """
             guard_not_invalidated(descr=...)
             p90 = newstr(4)
-            call(ConstClass(copy_raw_to_string), i55, p90, 0, 4, descr=)
+            call(ConstClass(copy_raw_to_string), i55, p90, 0, 4, descr=)
             guard_no_exception(descr=...)
             i91 = strgetitem(p90, 0)
             i92 = strgetitem(p90, 1)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -355,7 +355,7 @@
             i17 = arraylen_gc(p15, descr=)
             i18 = int_lt(i17, i15)
             # a cond call to _ll_list_resize_hint_really_look_inside_iff
-            cond_call(i18, _, p8, i15, 1, descr=)
+            cond_call(i18, _, p8, i15, 1, descr=)
             guard_no_exception(descr=...)
             p17 = getfield_gc(p8, descr=)
             setarrayitem_gc(p17, i13, i12, descr=)
@@ -395,7 +395,7 @@
             setarrayitem_gc(p24, 0, p26, descr=)
             setfield_gc(p22, p24, descr=)
             }}}
-            p32 = call_may_force(_, p18, p22, descr=)
+            p32 = call_may_force(_, p18, p22, descr=)
             ...
         """)
 
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -74,7 +74,7 @@
             setfield_gc(p13, p15, descr=)
             setfield_gc(p13, ConstPtr(0), descr=)
             }}}
-            i17 = call(ConstClass(ll_dict_lookup_trampoline), p13, p10, i12, 1, descr=)
+            i17 = call(ConstClass(ll_dict_lookup_trampoline), p13, p10, i12, 1, descr=)
             {{{
             setfield_gc(p13, 0, descr=)
             setfield_gc(p13, 0, descr=)
@@ -82,10 +82,10 @@
             }}}
             guard_no_exception(descr=...)
             p20 = new_with_vtable(ConstClass(W_IntObject))
-            call(ConstClass(_ll_dict_setitem_lookup_done_trampoline), p13, p10, p20, i12, i17, descr=)
+            call(ConstClass(_ll_dict_setitem_lookup_done_trampoline), p13, p10, p20, i12, i17, descr=)
             setfield_gc(p20, i5, descr=)
             guard_no_exception(descr=...)
-            i23 = call(ConstClass(ll_call_lookup_function), p13, p10, i12, 0, descr=)
+            i23 = call(ConstClass(ll_call_lookup_function), p13, p10, i12, 0, descr=)
             guard_no_exception(descr=...)
             i27 = int_lt(i23, 0)
             guard_false(i27, descr=...)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -35,7 +35,7 @@
             guard_not_invalidated(descr=...)
             i17 = force_token()
             setfield_gc(p0, i17, descr=<.* .*PyFrame.vable_token .*>)
-            f21 = call_release_gil(%s, 2.000000, 3.000000, descr=)
+            f21 = call_release_gil(%s, 2.000000, 3.000000, descr=)
             guard_not_forced(descr=...)
             guard_no_exception(descr=...)
         """ % pow_addr)
@@ -202,7 +202,7 @@
         assert loop.match_by_id('cfficall', """
             p96 = force_token()
             setfield_gc(p0, p96, descr=)
-            f97 = call_release_gil(91, i59, 1.0, 3, descr=)
+            f97 = call_release_gil(91, i59, 1.0, 3, descr=)
             guard_not_forced(descr=...)
             guard_no_exception(descr=...)
         """, ignore_ops=['guard_not_invalidated'])
@@ -266,7 +266,7 @@
         loop, = log.loops_by_id('cfficall')
         assert loop.match_by_id('cfficall', """
             ...
-            f1 = call_release_gil(..., descr=)
+            f1 = call_release_gil(..., descr=)
             ...
         """)
 
@@ -330,14 +330,14 @@
         guard_value(p165, ConstPtr(ptr70), descr=...)
         p166 = getfield_gc(p165, descr=)
         guard_value(p166, ConstPtr(ptr72), descr=...)
-        p167 = call(ConstClass(_ll_0_alloc_with_del___), descr=)
+        p167 = call(ConstClass(_ll_0_alloc_with_del___), descr=)
         guard_no_exception(descr=...)
         i112 = int_signext(i160, 2)
         setfield_gc(p167, ConstPtr(ptr85), descr=)
         i114 = int_ne(i160, i112)
         guard_false(i114, descr=...)
         --TICK--
-        i119 = call(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=)
+        i119 = call(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=)
         raw_store(i119, 0, i160, descr=)
         raw_store(i119, 2, i160, descr=)
         raw_store(i119, 4, i160, descr=)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -6,7 +6,7 @@
 else:
     SHIFT = 63
 
-# XXX review the  descrs to replace some EF=4 with EF=3 (elidable)
+# XXX review the  descrs to replace some EF=5 with EF=4 (elidable)
 
 
 class TestString(BaseTestPyPyC):
@@ -80,12 +80,12 @@
             i23 = strgetitem(p10, i19)
             p25 = newstr(1)
             strsetitem(p25, 0, i23)
-            p93 = call(ConstClass(fromstr), p25, 16, descr=)
+            p93 = call(ConstClass(fromstr), p25, 16, descr=)
             guard_no_exception(descr=...)
             i95 = getfield_gc_pure(p93, descr=)
             i96 = int_gt(i95, #)
             guard_false(i96, descr=...)
-            i94 = call(ConstClass(rbigint._toint_helper), p93, descr=)
+            i94 = call(ConstClass(rbigint._toint_helper), p93, descr=)
             guard_no_exception(descr=...)
             i95 = int_add_ovf(i6, i94)
             guard_no_overflow(descr=...)
@@ -119,21 +119,21 @@
             setfield_gc(p86, 23, descr=)
             setfield_gc(p86, 23, descr=)
             }}}
-            call(ConstClass(ll_append_res0__stringbuilderPtr_rpy_stringPtr), p86, p80, descr=)
+            call(ConstClass(ll_append_res0__stringbuilderPtr_rpy_stringPtr), p86, p80, descr=)
             guard_no_exception(descr=...)
             i89 = getfield_gc(p86, descr=)
             i90 = getfield_gc(p86, descr=)
             i91 = int_eq(i89, i90)
-            cond_call(i91, ConstClass(ll_grow_by__stringbuilderPtr_Signed), p86, 1, descr=)
+            cond_call(i91, ConstClass(ll_grow_by__stringbuilderPtr_Signed), p86, 1, descr=)
             guard_no_exception(descr=...)
             i92 = getfield_gc(p86, descr=)
             i93 = int_add(i92, 1)
             p94 = getfield_gc(p86, descr=)
             strsetitem(p94, i92, 32)
             setfield_gc(p86, i93, descr=)
-            call(ConstClass(ll_append_res0__stringbuilderPtr_rpy_stringPtr), p86, p80, descr=)
+            call(ConstClass(ll_append_res0__stringbuilderPtr_rpy_stringPtr), p86, p80, descr=)
             guard_no_exception(descr=...)
-            p95 = call(..., descr=)     # ll_build
+            p95 = call(..., descr=)     # ll_build
             guard_no_exception(descr=...)
             i96 = strlen(p95)
             i97 = int_add_ovf(i71, i96)
@@ -248,7 +248,7 @@
         i50 = int_add(i47, 1)
         setfield_gc(p15, i50, descr=)
         guard_not_invalidated(descr=...)
-        p52 = call(ConstClass(str_decode_ascii__raise_unicode_exception_decode), ConstPtr(ptr38), 3, 1, descr=)
+        p52 = call(ConstClass(str_decode_ascii__raise_unicode_exception_decode), ConstPtr(ptr38), 3, 1, descr=)
         guard_no_exception(descr=...)
         p53 = getfield_gc_pure(p52, descr=)
         guard_nonnull(p53, descr=...)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py
--- a/pypy/module/pypyjit/test_pypy_c/test_thread.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py
@@ -64,7 +64,7 @@
         guard_true(i56, descr=...)
         p57 = force_token()
         setfield_gc(p0, p57, descr=)
-        i58 = call_release_gil(0, _, i37, 1, descr=)
+        i58 = call_release_gil(0, _, i37, 1, descr=)
         guard_not_forced(descr=...)
         guard_no_exception(descr=...)
         i58 = int_sub(i44, 1)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_weakref.py b/pypy/module/pypyjit/test_pypy_c/test_weakref.py
--- a/pypy/module/pypyjit/test_pypy_c/test_weakref.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_weakref.py
@@ -35,7 +35,7 @@
         guard_nonnull_class(p66, ..., descr=...)
         p67 = force_token()
         setfield_gc(p0, p67, descr=)
-        p68 = call_may_force(ConstClass(WeakrefLifelineWithCallbacks.make_weakref_with_callback), p66, ConstPtr(ptr50), p14, ConstPtr(ptr51), descr=)
+        p68 = call_may_force(ConstClass(WeakrefLifelineWithCallbacks.make_weakref_with_callback), p66, ConstPtr(ptr50), p14, ConstPtr(ptr51), descr=)
         guard_not_forced(descr=...)
         guard_no_exception(descr=...)
         guard_nonnull_class(p68, ..., descr=...)
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -642,7 +642,7 @@
 
         thread_id = thread.get_ident()
         def other_thread():
-            print "thread started"
+            #print "thread started"
             lock2.release()
             lock1.acquire()
         lock1 = thread.allocate_lock()
diff --git a/pypy/module/test_lib_pypy/test_sqlite3.py b/pypy/module/test_lib_pypy/test_sqlite3.py
--- a/pypy/module/test_lib_pypy/test_sqlite3.py
+++ b/pypy/module/test_lib_pypy/test_sqlite3.py
@@ -276,6 +276,30 @@
         exc = raises(ValueError, cur.execute, "select 2\0")
         assert str(exc.value) == "the query contains a null character"
 
+    def test_close_in_del_ordering(self):
+        import gc
+        class SQLiteBackend(object):
+            success = False
+            def __init__(self):
+                self.connection = _sqlite3.connect(":memory:")
+            def close(self):
+                self.connection.close()
+            def __del__(self):
+                self.close()
+                SQLiteBackend.success = True
+            def create_db_if_needed(self):
+                conn = self.connection
+                cursor = conn.cursor()
+                cursor.execute("""
+                    create table if not exists nameoftable(value text)
+                """)
+                cursor.close()
+                conn.commit()
+        SQLiteBackend().create_db_if_needed()
+        gc.collect()
+        gc.collect()
+        assert SQLiteBackend.success
+
 
 class TestSQLiteHost(BaseTestSQLite):
     def setup_class(cls):
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -129,7 +129,8 @@
     # This is meant to be a *mixin*.
 
     def is_data_descr(space, w_obj):
-        return space.lookup(w_obj, '__set__') is not None
+        return (space.lookup(w_obj, '__set__') is not None or
+                space.lookup(w_obj, '__delete__') is not None)
 
     def get_and_call_args(space, w_descr, w_obj, args):
         # a special case for performance and to avoid infinite recursion
diff --git a/pypy/objspace/fake/checkmodule.py b/pypy/objspace/fake/checkmodule.py
--- a/pypy/objspace/fake/checkmodule.py
+++ b/pypy/objspace/fake/checkmodule.py
@@ -2,16 +2,19 @@
 from pypy.config.pypyoption import get_pypy_config
 
 
-def checkmodule(*modnames):
+def checkmodule(*modnames, **kwds):
+    translate_startup = kwds.pop('translate_startup', True)
+    assert not kwds
     config = get_pypy_config(translating=True)
     space = FakeObjSpace(config)
     seeobj_w = []
+    modules = []
     for modname in modnames:
         mod = __import__('pypy.module.%s' % modname, None, None, ['__doc__'])
         # force computation and record what we wrap
         module = mod.Module(space, W_Root())
         module.setup_after_space_initialization()
-        module.startup(space)
+        modules.append(module)
         for name in module.loaders:
             seeobj_w.append(module._load_lazily(space, name))
         if hasattr(module, 'submodules'):
@@ -20,5 +23,11 @@
                 for name in submod.loaders:
                     seeobj_w.append(submod._load_lazily(space, name))
     #
-    space.translates(seeobj_w=seeobj_w,
+    def func():
+        for mod in modules:
+            mod.startup(space)
+    if not translate_startup:
+        func()   # call it now
+        func = None
+    space.translates(func, seeobj_w=seeobj_w,
                      **{'translation.list_comprehension_operations': True})
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -113,7 +113,7 @@
 
 BUILTIN_TYPES = ['int', 'str', 'float', 'long', 'tuple', 'list', 'dict',
                  'unicode', 'complex', 'slice', 'bool', 'basestring', 'object',
-                 'bytearray', 'buffer']
+                 'bytearray', 'buffer', 'set', 'frozenset']
 
 class FakeObjSpace(ObjSpace):
     def __init__(self, config=None):
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -439,12 +439,12 @@
             try:
                 if encoding == 'ascii':
                     u = space.unicode_w(w_object)
-                    eh = unicodehelper.rpy_encode_error_handler()
+                    eh = unicodehelper.raise_unicode_exception_encode
                     return space.wrap(unicode_encode_ascii(
                             u, len(u), None, errorhandler=eh))
                 if encoding == 'utf-8':
                     u = space.unicode_w(w_object)
-                    eh = unicodehelper.rpy_encode_error_handler()
+                    eh = unicodehelper.raise_unicode_exception_encode
                     return space.wrap(unicode_encode_utf_8(
                             u, len(u), None, errorhandler=eh,
                             allow_surrogates=True))
diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py
--- a/pypy/objspace/test/test_descroperation.py
+++ b/pypy/objspace/test/test_descroperation.py
@@ -607,6 +607,18 @@
 
         raises(AttributeError, lambda: A().a)
 
+    def test_delete_descriptor(self):
+        class Prop(object):
+            def __get__(self, obj, cls):
+                return 42
+            def __delete__(self, obj):
+                obj.deleted = True
+        class C(object):
+            x = Prop()
+        obj = C()
+        del obj.x
+        assert obj.deleted
+
     def test_non_callable(self):
         meth = classmethod(1).__get__(1)
         raises(TypeError, meth)
diff --git a/pypy/tool/gdb_pypy.py b/pypy/tool/gdb_pypy.py
--- a/pypy/tool/gdb_pypy.py
+++ b/pypy/tool/gdb_pypy.py
@@ -5,7 +5,7 @@
 
 Or, alternatively:
 
-(gdb) python execfile('/path/to/gdb_pypy.py')
+(gdb) python exec(open('/path/to/gdb_pypy.py').read())
 """
 
 import re
@@ -55,10 +55,10 @@
 
 class RPyType(Command):
     """
-    Prints the RPython type of the expression (remember to dereference it!)
+    Prints the RPython type of the expression.
     E.g.:
 
-    (gdb) rpy_type *l_v123
+    (gdb) rpy_type l_v123
     GcStruct pypy.foo.Bar { super, inst_xxx, inst_yyy }
     """
 
@@ -73,23 +73,34 @@
 
     def invoke(self, arg, from_tty):
         # some magic code to automatically reload the python file while developing
-        from pypy.tool import gdb_pypy
         try:
-            reload(gdb_pypy)
+            from pypy.tool import gdb_pypy
+            try:
+                reload(gdb_pypy)
+            except:
+                import imp
+                imp.reload(gdb_pypy)
+            gdb_pypy.RPyType.prog2typeids = self.prog2typeids # persist the cache
+            self.__class__ = gdb_pypy.RPyType
+            result = self.do_invoke(arg, from_tty)
+            if not isinstance(result, str):
+                result = result.decode('latin-1')
+            print(result)
         except:
-            import imp
-            imp.reload(gdb_pypy)
-        gdb_pypy.RPyType.prog2typeids = self.prog2typeids # persist the cache
-        self.__class__ = gdb_pypy.RPyType
-        print (self.do_invoke(arg, from_tty).decode('latin-1'))
+            import traceback
+            traceback.print_exc()
 
     def do_invoke(self, arg, from_tty):
         try:
             offset = int(arg)
         except ValueError:
             obj = self.gdb.parse_and_eval(arg)
+            if obj.type.code == self.gdb.TYPE_CODE_PTR:
+                obj = obj.dereference()
             hdr = lookup(obj, '_gcheader')
             tid = hdr['h_tid']
+            if tid == -42:      # forwarded?
+                return 'Forwarded'
             if sys.maxsize < 2**32:
                 offset = tid & 0xFFFF     # 32bit
             else:
@@ -100,7 +111,7 @@
         if offset in typeids:
             return typeids[offset]
         else:
-            return 'Cannot find the type with offset %d' % offset
+            return 'Cannot find the type with offset 0x%x' % offset
 
     def get_typeids(self):
         try:
@@ -127,7 +138,7 @@
         try:
             self.gdb.execute('dump binary memory %s %s %s+%d' %
                              (fname, vstart, vstart, length))
-            with open(fname, 'rt') as fobj:
+            with open(fname, 'rb') as fobj:
                 data = fobj.read()
             return TypeIdsMap(zlib.decompress(data).splitlines(True), self.gdb)
         finally:
@@ -242,6 +253,8 @@
     fields
     """
 
+    recursive = False
+
     def __init__(self, val):
         self.val = val
 
@@ -249,29 +262,47 @@
     def lookup(cls, val, gdb=None):
         t = val.type
         if (is_ptr(t, gdb) and t.target().tag is not None and
-            re.match(r'pypy_list\d*', t.target().tag)):
+            re.match(r'pypy_(list|array)\d*', t.target().tag)):
             return cls(val)
         return None
 
     def to_string(self):
-        length = int(self.val['l_length'])
-        array = self.val['l_items']
-        allocated = int(array['length'])
-        items = array['items']
-        itemlist = []
-        for i in range(length):
-            item = items[i]
-            itemlist.append(str(item))
-        str_items = ', '.join(itemlist)
-        return 'r[%s] (len=%d, alloc=%d)' % (str_items, length, allocated)
+        t = self.val.type
+        if t.target().tag.startswith(r'pypy_array'):
+            if not self.val:
+                return 'r(null_array)'
+            length = int(self.val['length'])
+            items = self.val['items']
+            allocstr = ''
+        else:
+            if not self.val:
+                return 'r(null_list)'
+            length = int(self.val['l_length'])
+            array = self.val['l_items']
+            allocated = int(array['length'])
+            items = array['items']
+            allocstr = ', alloc=%d' % allocated
+        if RPyListPrinter.recursive:
+            str_items = '...'
+        else:
+            RPyListPrinter.recursive = True
+            try:
+                itemlist = []
+                for i in range(length):
+                    item = items[i]
+                    itemlist.append(str(item))    # may recurse here
+                str_items = ', '.join(itemlist)
+            finally:
+                RPyListPrinter.recursive = False
+        return 'r[%s] (len=%d%s)' % (str_items, length, allocstr)
 
 
 try:
     import gdb
     RPyType() # side effects
-    gdb.pretty_printers += [
+    gdb.pretty_printers = [
         RPyStringPrinter.lookup,
         RPyListPrinter.lookup
-        ]
+        ] + gdb.pretty_printers
 except ImportError:
     pass
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -65,6 +65,12 @@
 add --without-{0} option to skip packaging binary CFFI extension.""".format(module)
             raise MissingDependenciesError(module)
 
+def pypy_runs(pypy_c, quiet=False):
+    kwds = {}
+    if quiet:
+        kwds['stderr'] = subprocess.PIPE
+    return subprocess.call([str(pypy_c), '-c', 'pass'], **kwds) == 0
+
 def create_package(basedir, options):
     retval = 0
     name = options.name
@@ -87,6 +93,8 @@
             ' Please compile pypy first, using translate.py,'
             ' or check that you gave the correct path'
             ' with --override_pypy_c' % pypy_c)
+    if not pypy_runs(pypy_c):
+        raise OSError("Running %r failed!" % (str(pypy_c),))
     if not options.no_cffi:
         try:
             create_cffi_import_libraries(pypy_c, options)
@@ -100,6 +108,15 @@
     libpypy_name = 'libpypy-c.so' if not sys.platform.startswith('darwin') else 'libpypy-c.dylib'
     libpypy_c = pypy_c.new(basename=libpypy_name)
     if libpypy_c.check():
+        # check that this libpypy_c is really needed
+        os.rename(str(libpypy_c), str(libpypy_c) + '~')
+        try:
+            if pypy_runs(pypy_c, quiet=True):
+                raise Exception("It seems that %r runs without needing %r.  "
+                                "Please check and remove the latter" %
+                                (str(pypy_c), str(libpypy_c)))
+        finally:
+            os.rename(str(libpypy_c) + '~', str(libpypy_c))
         binaries.append((libpypy_c, libpypy_name))
     #
     builddir = options.builddir
diff --git a/pypy/tool/test/test_gdb_pypy.py b/pypy/tool/test/test_gdb_pypy.py
--- a/pypy/tool/test/test_gdb_pypy.py
+++ b/pypy/tool/test/test_gdb_pypy.py
@@ -204,6 +204,22 @@
     mylist.type.target().tag = None
     assert gdb_pypy.RPyListPrinter.lookup(mylist, FakeGdb) is None
 
+def test_pprint_array():
+    d = {'_gcheder': {'h_tid': 234}, 'length': 3, 'items': [20, 21, 22]}
+    mylist = PtrValue(d, type_tag='pypy_array1')
+    printer = gdb_pypy.RPyListPrinter.lookup(mylist, FakeGdb)
+    assert printer.to_string() == 'r[20, 21, 22] (len=3)'
+
+def test_pprint_null_list():
+    mylist = PtrValue({}, type_tag='pypy_list1')
+    printer = gdb_pypy.RPyListPrinter.lookup(mylist, FakeGdb)
+    assert printer.to_string() == 'r(null_list)'
+
+def test_pprint_null_array():
+    mylist = PtrValue({}, type_tag='pypy_array1')
+    printer = gdb_pypy.RPyListPrinter.lookup(mylist, FakeGdb)
+    assert printer.to_string() == 'r(null_array)'
+
 def test_typeidsmap():
     gdb = FakeGdb('', {exprmember(1): 111,
                        exprmember(2): 222,
diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -10,7 +10,6 @@
     Variable, Constant, FunctionGraph, checkgraph)
 from rpython.translator import simplify, transform
 from rpython.annotator import model as annmodel, signature
-from rpython.annotator.argument import simple_args
 from rpython.annotator.bookkeeper import Bookkeeper
 from rpython.rtyper.normalizecalls import perform_normalizations
 
@@ -91,22 +90,14 @@
 
     def get_call_parameters(self, function, args_s, policy):
         desc = self.bookkeeper.getdesc(function)
-        args = simple_args(args_s)
-        result = []
-        def schedule(graph, inputcells):
-            result.append((graph, inputcells))
-            return annmodel.s_ImpossibleValue
-
         prevpolicy = self.policy
         self.policy = policy
         self.bookkeeper.enter(None)
         try:
-            desc.pycall(schedule, args, annmodel.s_ImpossibleValue)
+            return desc.get_call_parameters(args_s)
         finally:
             self.bookkeeper.leave()
             self.policy = prevpolicy
-        [(graph, inputcells)] = result
-        return graph, inputcells
 
     def annotate_helper(self, function, args_s, policy=None):
         if policy is None:
diff --git a/rpython/annotator/argument.py b/rpython/annotator/argument.py
--- a/rpython/annotator/argument.py
+++ b/rpython/annotator/argument.py
@@ -155,18 +155,6 @@
         keywords_w = [_kwds_w[key] for key in self.keywords]
         return ArgumentsForTranslation(args_w, dict(zip(self.keywords, keywords_w)))
 
-    @classmethod
-    def fromshape(cls, (shape_cnt, shape_keys, shape_star), data_w):
-        args_w = data_w[:shape_cnt]
-        p = end_keys = shape_cnt + len(shape_keys)
-        if shape_star:
-            w_star = data_w[p]
-            p += 1
-        else:
-            w_star = None
-        return cls(args_w, dict(zip(shape_keys, data_w[shape_cnt:end_keys])),
-                w_star)
-
 
 def rawshape(args):
     return args._rawshape()
diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py
--- a/rpython/annotator/description.py
+++ b/rpython/annotator/description.py
@@ -1,13 +1,14 @@
 from __future__ import absolute_import
 import types
 from rpython.annotator.signature import (
-    enforce_signature_args, enforce_signature_return)
+    enforce_signature_args, enforce_signature_return, finish_type)
 from rpython.flowspace.model import Constant, FunctionGraph
 from rpython.flowspace.bytecode import cpython_code_signature
-from rpython.annotator.argument import rawshape, ArgErr
+from rpython.annotator.argument import rawshape, ArgErr, simple_args
 from rpython.tool.sourcetools import valid_identifier, func_with_new_name
 from rpython.tool.pairtype import extendabletype
-from rpython.annotator.model import AnnotatorError, SomeInteger, SomeString
+from rpython.annotator.model import (
+    AnnotatorError, SomeInteger, SomeString, s_ImpossibleValue)
 
 class CallFamily(object):
     """A family of Desc objects that could be called from common call sites.
@@ -75,7 +76,6 @@
         try:
             return self.attrs[attrname]
         except KeyError:
-            from rpython.annotator.model import s_ImpossibleValue
             return s_ImpossibleValue
 
     def set_s_value(self, attrname, s_value):
@@ -97,7 +97,6 @@
     # ClassAttrFamily is more precise: it is only about one attribut name.
 
     def __init__(self, desc):
-        from rpython.annotator.model import s_ImpossibleValue
         self.descs = {desc: True}
         self.read_locations = {}     # set of position_keys
         self.s_value = s_ImpossibleValue    # union of possible values
@@ -321,6 +320,24 @@
         result = unionof(result, s_previous_result)
         return result
 
+    def get_call_parameters(self, args_s):
+        args = simple_args(args_s)
+        inputcells = self.parse_arguments(args)
+        graph = self.specialize(inputcells)
+        assert isinstance(graph, FunctionGraph)
+        # if that graph has a different signature, we need to re-parse
+        # the arguments.
+        # recreate the args object because inputcells may have been changed
+        new_args = args.unmatch_signature(self.signature, inputcells)
+        inputcells = self.parse_arguments(new_args, graph)
+        signature = getattr(self.pyobj, '_signature_', None)
+        if signature:
+            s_result = finish_type(signature[1], self.bookkeeper, self.pyobj)
+            if s_result is not None:
+                self.bookkeeper.annotator.addpendingblock(
+                    graph, graph.returnblock, [s_result])
+        return graph, inputcells
+
     def bind_under(self, classdef, name):
         # XXX static methods
         return self.bookkeeper.getmethoddesc(self,
@@ -352,7 +369,6 @@
     @staticmethod
     def row_to_consider(descs, args, op):
         # see comments in CallFamily
-        from rpython.annotator.model import s_ImpossibleValue
         row = {}
         for desc in descs:
             def enlist(graph, ignore):
@@ -685,7 +701,6 @@
         # look up an attribute in the class
         cdesc = self.lookup(name)
         if cdesc is None:
-            from rpython.annotator.model import s_ImpossibleValue
             return s_ImpossibleValue
         else:
             # delegate to s_get_value to turn it into an annotation
@@ -999,7 +1014,6 @@
         try:
             value = self.read_attribute(attr)
         except AttributeError:
-            from rpython.annotator.model import s_ImpossibleValue
             return s_ImpossibleValue
         else:
             return self.bookkeeper.immutablevalue(value)
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -354,6 +354,25 @@
         assert isinstance(s, annmodel.SomeInteger)
         assert s.const == 3
 
+    def test_star_unpack_list(self):
+        def g():
+            pass
+        def f(l):
+            return g(*l)
+        a = self.RPythonAnnotator()
+        with py.test.raises(annmodel.AnnotatorError):
+            a.build_types(f, [[int]])
+
+    def test_star_unpack_and_keywords(self):
+        def g(a, b, c=0, d=0):
+            return a + b + c + d
+
+        def f(a, b):
+            return g(a, *(b,), d=5)
+        a = self.RPythonAnnotator()
+        s_result = a.build_types(f, [int, int])
+        assert isinstance(s_result, annmodel.SomeInteger)
+
     def test_pbc_attr_preserved_on_instance(self):
         a = self.RPythonAnnotator()
         s = a.build_types(snippet.preserve_pbc_attr_on_instance, [bool])
diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -6,6 +6,7 @@
 
 from rpython.flowspace.operation import op
 from rpython.flowspace.model import const, Constant
+from rpython.flowspace.argument import CallSpec
 from rpython.annotator.model import (SomeObject, SomeInteger, SomeBool,
     SomeString, SomeChar, SomeList, SomeDict, SomeTuple, SomeImpossibleValue,
     SomeUnicodeCodePoint, SomeInstance, SomeBuiltin, SomeBuiltinMethod,
@@ -47,11 +48,35 @@
 
 @op.simple_call.register(SomeObject)
 def simple_call_SomeObject(annotator, func, *args):
-    return annotator.annotation(func).call(simple_args([annotator.annotation(arg) for arg in args]))
+    return annotator.annotation(func).call(
+        simple_args([annotator.annotation(arg) for arg in args]))
+
+ at op.call_args.register_transform(SomeObject)
+def transform_varargs(annotator, v_func, v_shape, *data_v):
+    callspec = CallSpec.fromshape(v_shape.value, list(data_v))
+    v_vararg = callspec.w_stararg
+    if callspec.w_stararg:
+        s_vararg = annotator.annotation(callspec.w_stararg)
+        if not isinstance(s_vararg, SomeTuple):
+            raise AnnotatorError(
+                "Calls like f(..., *arg) require 'arg' to be a tuple")
+        n_items = len(s_vararg.items)
+        ops = [op.getitem(v_vararg, const(i)) for i in range(n_items)]
+        new_args = callspec.arguments_w + [hlop.result for hlop in ops]
+        if callspec.keywords:
+            newspec = CallSpec(new_args, callspec.keywords)
+            shape, data_v = newspec.flatten()
+            call_op = op.call_args(v_func, const(shape), *data_v)
+        else:
+            call_op = op.simple_call(v_func, *new_args)
+        ops.append(call_op)
+        return ops
+
 
 @op.call_args.register(SomeObject)
-def call_args(annotator, func, *args):
-    return annotator.annotation(func).call(complex_args([annotator.annotation(arg) for arg in args]))
+def call_args(annotator, func, *args_v):
+    callspec = complex_args([annotator.annotation(v_arg) for v_arg in args_v])
+    return annotator.annotation(func).call(callspec)
 
 class __extend__(SomeObject):
 
@@ -476,12 +501,18 @@
         return SomeInteger(nonneg=True)
 
     def method_strip(self, chr=None):
+        if chr is None and isinstance(self, SomeUnicodeString):
+            raise AnnotatorError("unicode.strip() with no arg is not RPython")
         return self.basestringclass(no_nul=self.no_nul)
 
     def method_lstrip(self, chr=None):
+        if chr is None and isinstance(self, SomeUnicodeString):
+            raise AnnotatorError("unicode.lstrip() with no arg is not RPython")
         return self.basestringclass(no_nul=self.no_nul)
 
     def method_rstrip(self, chr=None):
+        if chr is None and isinstance(self, SomeUnicodeString):
+            raise AnnotatorError("unicode.rstrip() with no arg is not RPython")
         return self.basestringclass(no_nul=self.no_nul)
 
     def method_join(self, s_list):
@@ -722,7 +753,7 @@
         if attr not in dct:
             continue
         obj = dct[attr]
-        if (not isinstance(obj, Constant) or 
+        if (not isinstance(obj, Constant) or
                 not isinstance(obj.value, property)):
             return
         result.append(getattr(obj.value, meth))
diff --git a/rpython/doc/rpython.rst b/rpython/doc/rpython.rst
--- a/rpython/doc/rpython.rst
+++ b/rpython/doc/rpython.rst
@@ -59,7 +59,7 @@
 
 **exceptions**
 
-  fully supported
+  fully supported.
   see below `Exception rules`_ for restrictions on exceptions raised by built-in operations
 
 
@@ -92,6 +92,11 @@
   no variable-length tuples; use them to store or return pairs or n-tuples of
   values. Each combination of types for elements and length constitute
   a separate and not mixable type.
+  
+  There is no general way to convert a list into a tuple, because the
+  length of the result would not be known statically.  (You can of course
+  do ``t = (lst[0], lst[1], lst[2])`` if you know that ``lst`` has got 3
+  items.)
 
 **lists**
 
@@ -136,9 +141,16 @@
 
 **functions**
 
-+ statically called functions may use defaults and a variable number of
-  arguments (which may be passed as a list instead of a tuple, so write code
-  that does not depend on it being a tuple).
++ function declarations may use defaults and ``*args``, but not
+  ``**keywords``.
+
++ function calls may be done to a known function or to a variable one,
+  or to a method.  You can call with positional and keyword arguments,
+  and you can pass a ``*args`` argument (it must be a tuple).
+
++ as explained above, tuples are not of a variable length.  If you need
+  to call a function with a dynamic number of arguments, refactor the
+  function itself to accept a single argument which is a regular list.
 
 + dynamic dispatch enforces the use of signatures that are equal for all
   possible called function, or at least "compatible enough".  This
diff --git a/rpython/flowspace/argument.py b/rpython/flowspace/argument.py
--- a/rpython/flowspace/argument.py
+++ b/rpython/flowspace/argument.py
@@ -111,3 +111,16 @@
             return self.arguments_w
         else:
             return self.arguments_w + [const(x) for x in self.w_stararg.value]
+
+    @classmethod
+    def fromshape(cls, (shape_cnt, shape_keys, shape_star), data_w):
+        args_w = data_w[:shape_cnt]
+        p = end_keys = shape_cnt + len(shape_keys)
+        if shape_star:
+            w_star = data_w[p]
+            p += 1
+        else:
+            w_star = None
+        return cls(args_w, dict(zip(shape_keys, data_w[shape_cnt:end_keys])),
+                w_star)
+
diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -31,6 +31,8 @@
             self.rtyper = cpu.rtyper
             translator = self.rtyper.annotator.translator
             self.raise_analyzer = RaiseAnalyzer(translator)
+            self.raise_analyzer_ignore_memoryerror = RaiseAnalyzer(translator)
+            self.raise_analyzer_ignore_memoryerror.do_ignore_memory_error()
             self.readwrite_analyzer = ReadWriteAnalyzer(translator)
             self.virtualizable_analyzer = VirtualizableAnalyzer(translator)
             self.quasiimmut_analyzer = QuasiImmutAnalyzer(translator)
@@ -141,7 +143,7 @@
     def grab_initial_jitcodes(self):
         for jd in self.jitdrivers_sd:
             jd.mainjitcode = self.get_jitcode(jd.portal_graph)
-            jd.mainjitcode.is_portal = True
+            jd.mainjitcode.jitdriver_sd = jd
 
     def enum_pending_graphs(self):
         while self.unfinished_graphs:
@@ -260,11 +262,14 @@
             elif loopinvariant:
                 extraeffect = EffectInfo.EF_LOOPINVARIANT
             elif elidable:
-                if self._canraise(op):
+                cr = self._canraise(op)
+                if cr == "mem":
+                    extraeffect = EffectInfo.EF_ELIDABLE_OR_MEMORYERROR
+                elif cr:
                     extraeffect = EffectInfo.EF_ELIDABLE_CAN_RAISE
                 else:
                     extraeffect = EffectInfo.EF_ELIDABLE_CANNOT_RAISE
-            elif self._canraise(op):
+            elif self._canraise(op):   # True or "mem"
                 extraeffect = EffectInfo.EF_CAN_RAISE
             else:
                 extraeffect = EffectInfo.EF_CANNOT_RAISE
@@ -278,6 +283,7 @@
                 " effects): EF=%s" % (op, extraeffect))
         if elidable:
             if extraeffect not in (EffectInfo.EF_ELIDABLE_CANNOT_RAISE,
+                                   EffectInfo.EF_ELIDABLE_OR_MEMORYERROR,
                                    EffectInfo.EF_ELIDABLE_CAN_RAISE):
                 raise Exception(
                 "in operation %r: this calls an _elidable_function_,"
@@ -301,10 +307,17 @@
                                     effectinfo)
 
     def _canraise(self, op):
+        """Returns True, False, or "mem" to mean 'only MemoryError'."""
         if op.opname == 'pseudo_call_cannot_raise':
             return False
         try:
-            return self.raise_analyzer.can_raise(op)
+            if self.raise_analyzer.can_raise(op):
+                if self.raise_analyzer_ignore_memoryerror.can_raise(op):
+                    return True
+                else:
+                    return "mem"
+            else:
+                return False
         except lltype.DelayedPointer:
             return True  # if we need to look into the delayed ptr that is
                          # the portal, then it's certainly going to raise
diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py
--- a/rpython/jit/codewriter/effectinfo.py
+++ b/rpython/jit/codewriter/effectinfo.py
@@ -11,10 +11,11 @@
     EF_ELIDABLE_CANNOT_RAISE           = 0 #elidable function (and cannot raise)
     EF_LOOPINVARIANT                   = 1 #special: call it only once per loop
     EF_CANNOT_RAISE                    = 2 #a function which cannot raise
-    EF_ELIDABLE_CAN_RAISE              = 3 #elidable function (but can raise)
-    EF_CAN_RAISE                       = 4 #normal function (can raise)
-    EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE = 5 #can raise and force virtualizables
-    EF_RANDOM_EFFECTS                  = 6 #can do whatever
+    EF_ELIDABLE_OR_MEMORYERROR         = 3 #elidable, can only raise MemoryError
+    EF_ELIDABLE_CAN_RAISE              = 4 #elidable function (but can raise)
+    EF_CAN_RAISE                       = 5 #normal function (can raise)
+    EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE = 6 #can raise and force virtualizables
+    EF_RANDOM_EFFECTS                  = 7 #can do whatever
 
     # the 'oopspecindex' field is one of the following values:
     OS_NONE                     = 0    # normal case, no oopspec
@@ -126,20 +127,27 @@
         if extraeffect == EffectInfo.EF_RANDOM_EFFECTS:
             assert readonly_descrs_fields is None
             assert readonly_descrs_arrays is None
+            assert readonly_descrs_interiorfields is None
             assert write_descrs_fields is None
             assert write_descrs_arrays is None
+            assert write_descrs_interiorfields is None
         else:
             assert readonly_descrs_fields is not None
             assert readonly_descrs_arrays is not None
+            assert readonly_descrs_interiorfields is not None
             assert write_descrs_fields is not None
             assert write_descrs_arrays is not None
+            assert write_descrs_interiorfields is not None
         result = object.__new__(cls)
         result.readonly_descrs_fields = readonly_descrs_fields
         result.readonly_descrs_arrays = readonly_descrs_arrays
         result.readonly_descrs_interiorfields = readonly_descrs_interiorfields
         if extraeffect == EffectInfo.EF_LOOPINVARIANT or \
            extraeffect == EffectInfo.EF_ELIDABLE_CANNOT_RAISE or \
+           extraeffect == EffectInfo.EF_ELIDABLE_OR_MEMORYERROR or \

From noreply at buildbot.pypy.org  Wed Apr 22 12:20:41 2015
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Wed, 22 Apr 2015 12:20:41 +0200 (CEST)
Subject: [pypy-commit] pypy default: Removed pdb,
	added warning message and made more robust.
Message-ID: <20150422102041.467B11C06CD@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: 
Changeset: r76868:4e43d96d9d48
Date: 2015-02-22 11:43 +0100
http://bitbucket.org/pypy/pypy/changeset/4e43d96d9d48/

Log:	Removed pdb, added warning message and made more robust.

diff --git a/rpython/rlib/rstrategies/logparser.py b/rpython/rlib/rstrategies/logparser.py
--- a/rpython/rlib/rstrategies/logparser.py
+++ b/rpython/rlib/rstrategies/logparser.py
@@ -12,7 +12,7 @@
 If the name of the logfile includes one of the AVAILABLE_VMS as a substring, the first three global variables
 are automatically configured.
 The script should work without these configurations, but the output will probably not be that pretty.
-To avoid errors, use the -a flag when running without proper configuration.
+To avoid errors, the -a flag is implied when running without proper configuration.
 """
 
 # This should contain a full list of storage nodes (strategies).
@@ -45,6 +45,21 @@
 
 AVAILABLE_VMS = ['RSqueak', 'Pycket', 'Topaz']
 
+def configure_vm(logfile, flags):
+    vm_config_name = None
+    for vm_name in AVAILABLE_VMS:
+        if vm_name in logfile:
+            vm_config_name = vm_name
+            break
+    if vm_config_name is not None:
+        print "Using VM configuration %s" % vm_name
+        SET_VM(vm_name)
+    else:
+        print "No VM configuration found in filename '%s'. Available configurations: %s" % \
+                (logfile, AVAILABLE_VMS)
+        print "Please add new VM configuration or rename logfile. Turning on -a flag to avoid errors."
+        flags.allstorage = True
+
 # ====================================================================
 # ======== Logfile parsing
 # ====================================================================
@@ -370,8 +385,6 @@
         self.operations = set()
     
     def node(self, name):
-        if str(name) == 'None':
-            import pdb; pdb.set_trace()
         if name not in self.nodes:
             self.nodes[name] = StorageNode(name)
         return self.nodes[name]
@@ -666,11 +679,7 @@
         usage(flags, commands)
     logfile = argv[0]
     flags.logfile = logfile
-    for vm_name in AVAILABLE_VMS:
-        if vm_name in logfile:
-            print "Using VM configuration %s" % vm_name
-            SET_VM(vm_name)
-            break
+    configure_vm(logfile, flags)
     command = argv[1]
     for flag in argv[2:]:
         if not flags.handle(flag):

From noreply at buildbot.pypy.org  Wed Apr 22 12:20:40 2015
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Wed, 22 Apr 2015 12:20:40 +0200 (CEST)
Subject: [pypy-commit] pypy default: Added rstrategies (from
 https://github.com/antongulenko/rstrategies).
Message-ID: <20150422102040.0BA2A1C06CD@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: 
Changeset: r76867:25ca52e41849
Date: 2015-02-15 19:15 +0100
http://bitbucket.org/pypy/pypy/changeset/25ca52e41849/

Log:	Added rstrategies (from
	https://github.com/antongulenko/rstrategies).

diff --git a/rpython/rlib/rstrategies/README.md b/rpython/rlib/rstrategies/README.md
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rstrategies/README.md
@@ -0,0 +1,101 @@
+# rstrategies
+
+A library to implement storage strategies in VMs based on the RPython toolchain.
+rstrategies can be used in VMs for any language or language family.
+
+This library has been developed as part of a Masters Thesis by [Anton Gulenko](https://github.com/antongulenko).
+
+The original paper describing the optimization "Storage Strategies for collections in dynamically typed languages" by C.F. Bolz, L. Diekmann and L. Tratt can be found [here](http://stups.hhu.de/mediawiki/images/3/3b/Pub-BoDiTr13_246.pdf).
+
+So far, this library has been adpoted by 3 VMs: [RSqueak](https://github.com/HPI-SWA-Lab/RSqueak), [Topaz](https://github.com/topazproject/topaz) ([Forked here](https://github.com/antongulenko/topaz/tree/rstrategies)) and [Pycket](https://github.com/samth/pycket) ([Forked here](https://github.com/antongulenko/pycket/tree/rstrategies)).
+
+#### Concept
+
+Collections are often used homogeneously, i.e. they contain only objects of the same type.
+Primitive numeric types like ints or floats are especially interesting for optimization.
+These cases can be optimized by storing the unboxed data of these objects in consecutive memory.
+This is done by letting a special "strategy" object handle the entire storage of a collection.
+The collection object holds two separate references: one to its strategy and one to its storage.
+Every operation on the collection is delegated to the strategy, which accesses the storage when needed.
+The strategy can be switched to a more suitable one, which might require converting the storage array.
+
+## Usage
+
+The following are the steps needed to integrated rstrategies in an RPython VM.
+Because of the special nature of this library it is not enough to simply call some API methods; the library must be integrated within existing VM classes using a metaclass, mixins and other meta-programming techniques.
+
+The sequence of steps described here is something like a "setup walkthrough", and might be a bit abstract.
+To see a concrete example, look at [AbstractShadow](https://github.com/HPI-SWA-Lab/RSqueak/blob/d5ff2572106d23a5246884de6f8b86f46d85f4f7/spyvm/storage.py#L73), [StrategyFactory](https://github.com/HPI-SWA-Lab/RSqueak/blob/d5ff2572106d23a5246884de6f8b86f46d85f4f7/spyvm/storage.py#L126) and [W_PointersObject](https://github.com/HPI-SWA-Lab/RSqueak/blob/d5ff2572106d23a5246884de6f8b86f46d85f4f7/spyvm/model.py#L565) from the [RSqueak VM](https://github.com/HPI-SWA-Lab/RSqueak).
+The code is also well commented.
+
+#### Basics
+
+Currently the rstrategies library supports fixed sized and variable sized collections.
+This can be used to optimize a wide range of primitive data structures like arrays, lists or regular objects.
+Any of these are called 'collections' in this context.
+The VM should have a central class or class hierarchy for collections.
+In order to extend these classes and use strategies, the library needs accessor methods for two attributes of collection objects: strategy and storage.
+The easiest way is adding the following line to the body of the root collection class:
+```
+rstrategies.make_accessors(strategy='strategy', storage='storage')
+```
+This will generate the 4 accessor methods ```_[get/set]_[storage/strategy]()``` for the respective attributes.
+Alternatively, implement these methods manually or overwrite the getters/setters in ```StrategyFactory```.
+
+Next, the strategy classes must be defined. This requires a small class hierarchy with a dedicated root class.
+In the definition of this root class, include the following lines:
+```
+    __metaclass__ = rstrategies.StrategyMetaclass
+    import_from_mixin(rstrategies.AbstractStrategy)
+    import_from_mixin(rstrategies.SafeIndexingMixin)
+```
+
+```import_from_mixin``` can be found in ```rpython.rlib.objectmodel```.
+If index-checking is performed safely at other places in the VM, you can use ```rstrategies.UnsafeIndexingMixin``` instead.
+If you need your own metaclass, you can combine yours with the rstrategies one using multiple inheritance [like here](https://github.com/HPI-SWA-Lab/RSqueak/blob/d5ff2572106d23a5246884de6f8b86f46d85f4f7/spyvm/storage_contexts.py#L24).
+Also implement a ```storage_factory()``` method, which returns an instance of ```rstrategies.StorageFactory```, which is described below.
+
+#### Strategy classes
+
+Now you can create the actual strategy classes, subclassing them from the single root class.
+The following list summarizes the basic strategies available.
+* ```EmptyStrategy```
+    A strategy for empty collections; very efficient, but limited. Does not allocate anything.
+* ```SingleValueStrategy```
+    A strategy for collections containing the same object ```n``` times. Only allocates memory to store the size of the collection.
+* ```GenericStrategy```
+    A non-optimized strategy backed by a generic python list. This is the fallback strategy, since it can store everything, but is not optimized.
+* ```WeakGenericStrategy```
+    Like ```GenericStrategy```, but uses ```weakref``` to hold on weakly to its elements.
+* ```SingleTypeStrategy```
+    Can store a single unboxed type like int or float. This is the main 
+* ```TaggingStrategy```
+    Extension of SingleTypeStrategy. Uses a specific value in the value range of the unboxed type to represent
+    one additional, arbitrary object.
+
+There are also intermediate classes, which allow creating new, more customized strategies. For this, you should get familiar with the code.
+
+Include one of these mixin classes using ```import_from_mixin```.
+The mixin classes contain comments describing methods or fields which are also required in the strategy class in order to use them.
+Additionally, add the @rstrategies.strategy(generalize=alist) decorator to all strategy classes.
+The ```alist``` parameter must contain all strategies, which the decorated strategy can switch to, if it can not represent a new element anymore.
+[Example](https://github.com/HPI-SWA-Lab/RSqueak/blob/d5ff2572106d23a5246884de6f8b86f46d85f4f7/spyvm/storage.py#L87) for an implemented strategy.
+See the other strategy classes behind this link for more examples.
+
+#### Strategy Factory
+
+The last part is subclassing ```rstrategies.StrategyFactory```, overwriting the method ```instantiate_strategy``` if necessary and passing the strategies root class to the constructor.
+The factory provides the methods ```switch_strategy```, ```set_initial_strategy```, ```strategy_type_for``` which can be used by the VM code to use the mechanism behind strategies.
+See the comments in the source code.
+
+The strategy mixins offer the following methods to manipulate the contents of the collection:
+* basic API
+    * ```size```
+* fixed size API
+    * ```store```, ```fetch```, ```slice```, ```store_all```, ```fetch_all```
+* variable size API
+    * ```insert```, ```delete```, ```append```, ```pop```
+
+If the collection has a fixed size, simply never use any of the variable size methods in the VM code.
+Since the strategies are singletons, these methods need the collection object as first parameter.
+For convenience, more fitting accessor methods should be implemented on the collection class itself.
diff --git a/rpython/rlib/rstrategies/__init__.py b/rpython/rlib/rstrategies/__init__.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rstrategies/__init__.py
@@ -0,0 +1,1 @@
+# Empy
diff --git a/rpython/rlib/rstrategies/logger.py b/rpython/rlib/rstrategies/logger.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rstrategies/logger.py
@@ -0,0 +1,54 @@
+
+class LogEntry(object):
+    def __init__(self):
+        self.slots = 0
+        self.objects = 0
+        self.element_typenames = {}
+        
+    def add(self, size, element_typename):
+        self.slots += size
+        self.objects += 1
+        if element_typename:
+            self.element_typenames[element_typename] = None
+    
+    def classnames(self):
+        return self.element_typenames.keys()
+
+class Logger(object):
+    _attrs_ = ["active", "aggregate", "logs"]
+    _immutable_fields_ = ["active?", "aggregate?", "logs"]
+    
+    def __init__(self):
+        self.active = False
+        self.aggregate = False
+        self.logs = {}
+    
+    def activate(self, aggregate=False):
+        self.active = True
+        self.aggregate = self.aggregate or aggregate
+    
+    def log(self, new_strategy, size, cause="", old_strategy="", typename="", element_typename=""):
+        if self.aggregate:
+            key = (cause, old_strategy, new_strategy, typename)
+            if key not in self.logs:
+                self.logs[key] = LogEntry()
+            entry = self.logs[key]
+            entry.add(size, element_typename)
+        else:
+            element_typenames = [ element_typename ] if element_typename else []
+            self.output(cause, old_strategy, new_strategy, typename, size, 1, element_typenames)
+    
+    def print_aggregated_log(self):
+        if not self.aggregate:
+            return
+        for key, entry in self.logs.items():
+            cause, old_strategy, new_strategy, typename = key
+            slots, objects, element_typenames = entry.slots, entry.objects, entry.classnames()
+            self.output(cause, old_strategy, new_strategy, typename, slots, objects, element_typenames)
+    
+    def output(self, cause, old_strategy, new_strategy, typename, slots, objects, element_typenames):
+        old_strategy_string = "%s -> " % old_strategy if old_strategy else ""
+        classname_string = " of %s" % typename if typename else ""
+        element_string = (" elements: " + " ".join(element_typenames)) if element_typenames else ""
+        format = (cause, old_strategy_string, new_strategy, classname_string, slots, objects, element_string)
+        print "%s (%s%s)%s size %d objects %d%s" % format
diff --git a/rpython/rlib/rstrategies/logparser.py b/rpython/rlib/rstrategies/logparser.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rstrategies/logparser.py
@@ -0,0 +1,685 @@
+
+import re, os, sys, operator
+
+"""
+This script parses a log produced by rstrategies_logger.py into a graph and converts it to various outputs.
+The most useful outputs are the dot* commands producing a visualization of the log using the dot-command of graphviz.
+Every strategy is a node in the graph, and the edges are collections or objects that transition between
+two strategies at some point during the log.
+Artificial nodes are created for log entries without an explicit source node. These are the events when a
+collection is created.
+The input to this script is a logfile, a command and optional flags.
+If the name of the logfile includes one of the AVAILABLE_VMS as a substring, the first three global variables
+are automatically configured.
+The script should work without these configurations, but the output will probably not be that pretty.
+To avoid errors, use the -a flag when running without proper configuration.
+"""
+
+# This should contain a full list of storage nodes (strategies).
+# All strategies not included here will be combined into a single "Other"-node, if the -a flag is not given.
+STORAGE_NODES = []
+
+# This allows arbitrary renamings of storage strategy nodes
+NODE_RENAMINGS = {}
+
+# Artificial storage-source nodes are automatically named like the associated operation.
+# This dict allows customizing the names of these nodes.
+STORAGE_SOURCES = {}
+
+def SET_VM(vm_name):
+    global STORAGE_NODES
+    global NODE_RENAMINGS
+    global STORAGE_SOURCES
+    if vm_name == 'RSqueak':
+        STORAGE_NODES = ['List', 'WeakList', 'SmallIntegerOrNil', 'FloatOrNil', 'AllNil']
+        NODE_RENAMINGS = dict((x+'Strategy', x) for x in STORAGE_NODES)
+        STORAGE_SOURCES = {'Filledin': 'Image Loading', 'Initialized': 'Object Creation'}
+    elif vm_name == 'Pycket':
+        STORAGE_SOURCES = {'Created': 'Array Creation'}
+        # TODO
+    elif vm_name == 'Topaz':
+        # TODO
+        pass
+    else:
+        raise Exception("Unhandled vm name %s" % vm_name)
+
+AVAILABLE_VMS = ['RSqueak', 'Pycket', 'Topaz']
+
+# ====================================================================
+# ======== Logfile parsing
+# ====================================================================
+
+def percent(part, total):
+    if total == 0:
+        return 0
+    return float(part)*100 / total
+
+def parse(filename, flags, callback):
+    parsed_entries = 0
+    if filename == "-":
+        opener = lambda: sys.stdin
+    else:
+        opener = lambda: open(filename, 'r', 1)
+    with opener() as file:
+        while True:
+            line = file.readline()
+            if len(line) == 0:
+                break
+            entry = parse_line(line, flags)
+            if entry:
+                parsed_entries += 1
+                callback(entry)
+    return parsed_entries
+
+line_pattern = re.compile("^(?P\w+) \(((?P\w+) -> )?(?P\w+)\)( of (?P.+))? size (?P[0-9]+)( objects (?P[0-9]+))?( elements: (?P.+( .+)*))?$")
+
+def parse_line(line, flags):
+    result = line_pattern.match(line)
+    if result is None:
+        if flags.verbose:
+            print "Could not parse line: %s" % line[:-1]
+        return None
+    operation = str(result.group('operation'))
+    old_storage = result.group('old')
+    new_storage = str(result.group('new'))
+    classname = str(result.group('classname'))
+    size = int(result.group('size'))
+    objects = result.group('objects')
+    objects = int(objects) if objects else 1
+    classnames = result.group('classnames')
+    if classnames is not None:
+        classnames = classnames.split(' ')
+        classnames = set(classnames)
+    else:
+        classnames = set()
+    
+    is_storage_source = old_storage is None
+    if is_storage_source:
+        if operation in STORAGE_SOURCES:
+            old_storage = STORAGE_SOURCES[operation]
+        else:
+            print "Using operation %s as storage source." % operation
+    old_storage = str(old_storage)
+    
+    if new_storage in NODE_RENAMINGS:
+        new_storage = NODE_RENAMINGS[new_storage]
+    if old_storage in NODE_RENAMINGS:
+        old_storage = NODE_RENAMINGS[old_storage]
+    
+    return LogEntry(operation, old_storage, new_storage, classname, size, objects, classnames, is_storage_source)
+
+class LogEntry(object):
+    
+    def __init__(self, operation, old_storage, new_storage, classname, size, objects, classnames, is_storage_source):
+        self.operation = operation
+        self.old_storage = old_storage
+        self.new_storage = new_storage
+        self.classname = classname
+        self.size = size
+        self.objects = objects
+        self.classnames = classnames
+        self.is_storage_source = is_storage_source
+        assert old_storage != new_storage, "old and new storage identical in log entry: %s" % self
+    
+    def full_key(self):
+        return (self.operation, self.old_storage, self.new_storage)
+    
+    def __lt__(self, other):
+        return self.classname < other.classname
+    
+    def __repr__(self):
+        return "%s(%s)" % (self.__str__(), object.__repr__(self))
+    
+    def __str__(self):
+        old_storage_string = "%s -> " % self.old_storage if self.old_storage else ""
+        classname_string = " of %s" % self.classname if self.classname else ""
+        objects_string = " objects %d" % self.objects if self.objects > 1 else ""
+        return "%s (%s%s)%s size %d%s" % (self.operation, old_storage_string, self.new_storage, classname_string, self.size, objects_string)
+
+# ====================================================================
+# ======== Graph parsing
+# ====================================================================
+
+class Operations(object):
+    
+    def __init__(self, objects=0, slots=0, element_classnames=[]):
+        self.objects = objects
+        self.slots = slots
+        self.element_classnames = set(element_classnames)
+    
+    def __str__(self, total=None):
+        if self.objects == 0:
+            avg_slots = 0
+        else:
+            avg_slots = float(self.slots) / self.objects
+        if total is not None and total.slots != 0:
+            percent_slots = " (%.1f%%)" % percent(self.slots, total.slots)
+        else:
+            percent_slots = ""
+        if total is not None and total.objects != 0:
+            percent_objects = " (%.1f%%)" % percent(self.objects, total.objects)
+        else:
+            percent_objects = ""
+        slots = format(self.slots, ",d")
+        objects = format(self.objects, ",d")
+        classnames = (" [ elements: %s ]" % ' '.join([str(x) for x in self.element_classnames])) \
+                                    if len(self.element_classnames) else ""
+        return "%s%s slots in %s%s objects (avg size: %.1f)%s" % (slots, percent_slots, objects, percent_objects, avg_slots, classnames)
+    
+    def __repr__(self):
+        return "%s(%s)" % (self.__str__(), object.__repr__(self))
+    
+    def add_log_entry(self, entry):
+        self.slots = self.slots + entry.size
+        self.objects = self.objects + entry.objects
+        self.element_classnames |= entry.classnames
+    
+    def __sub__(self, other):
+        return Operations(self.objects - other.objects, self.slots - other.slots)
+    
+    def __add__(self, other):
+        return Operations(self.objects + other.objects, self.slots + other.slots)
+    
+    def __lt__(self, other):
+        return self.slots < other.slots
+    
+    def empty(self):
+        return self.objects == 0 and self.slots == 0
+    
+    def prefixprint(self, key="", total=None):
+        if not self.empty():
+            print "%s%s" % (key, self.__str__(total))
+    
+class ClassOperations(object):
+    
+    def __init__(self):
+        self.classes = {}
+    
+    def cls(self, name):
+        if name not in self.classes:
+            self.classes[name] = Operations()
+        return self.classes[name]
+    
+    def total(self):
+        return reduce(operator.add, self.classes.values(), Operations())
+    
+    def __str__(self):
+        return "ClassOperations(%s)" % self.classes
+    
+    def __repr__(self):
+        return "%s(%s)" % (self.__str__(), object.__repr__(self))
+    
+    def __add__(self, other):
+        result = ClassOperations()
+        result.classes = dict(self.classes)
+        for classname, other_class in other.classes.items():
+            result.cls(classname) # Make sure exists.
+            result.classes[classname] += other_class
+        return result
+    
+    def __sub__(self, other):
+        result = ClassOperations()
+        result.classes = dict(self.classes)
+        for classname, other_class in other.classes.items():
+            result.cls(classname) # Make sure exists.
+            result.classes[classname] -= other_class
+        return result
+    
+class StorageEdge(object):
+    
+    def __init__(self, operation="None", origin=None, target=None):
+        self.operation = operation
+        self.classes = ClassOperations()
+        self.origin = origin
+        self.target = target
+        self.is_storage_source = False
+    
+    def full_key(self):
+        return (self.operation, self.origin.name, self.target.name)
+    
+    def cls(self, classname):
+        return self.classes.cls(classname)
+    
+    def total(self):
+        return self.classes.total()
+    
+    def notify_nodes(self):
+        self.origin.note_outgoing(self)
+        self.target.note_incoming(self)
+    
+    def add_log_entry(self, entry):
+        self.cls(entry.classname).add_log_entry(entry)
+        if entry.is_storage_source:
+            self.is_storage_source = True
+    
+    def as_log_entries(self):
+        entries = []
+        for classname, ops in self.classes.classes.items():
+            origin = None if self.is_storage_source else self.origin.name
+            entry = LogEntry(self.operation, origin, self.target.name, classname,
+                            ops.slots, ops.objects, ops.element_classnames, self.is_storage_source)
+            entries.append(entry)
+        return entries
+    
+    def __lt__(self, other):
+        return self.full_key() < other.full_key()
+    
+    def __str__(self):
+        return "[%s %s -> %s]" % (self.operation, self.origin, self.target)
+    
+    def __repr__(self):
+        return "%s(%s)" % (self.__str__(), object.__repr__(self))
+    
+    def __add__(self, other):
+        origin = self.origin if self.origin is not None else other.origin
+        target = self.target if self.target is not None else other.target
+        result = StorageEdge(self.operation, origin, target)
+        result.classes += self.classes + other.classes
+        return result
+    
+    def __sub__(self, other):
+        origin = self.origin if self.origin is not None else other.origin
+        target = self.target if self.target is not None else other.target
+        result = StorageEdge(self.operation, origin, target)
+        result.classes += self.classes - other.classes
+        return result
+    
+class StorageNode(object):
+    
+    def __init__(self, name):
+        self.name = name
+        self.incoming = set()
+        self.outgoing = set()
+    
+    def note_incoming(self, edge):
+        assert edge.target is self
+        if edge not in self.incoming:
+            self.incoming.add(edge)
+        
+    def note_outgoing(self, edge):
+        assert edge.origin is self
+        if edge not in self.outgoing:
+            self.outgoing.add(edge)
+        
+    def incoming_edges(self, operation):
+        return filter(lambda x: x.operation == operation, self.incoming)
+    
+    def outgoing_edges(self, operation):
+        return filter(lambda x: x.operation == operation, self.outgoing)
+    
+    def sum_incoming(self, operation):
+        return reduce(operator.add, self.incoming_edges(operation), StorageEdge(operation))
+        
+    def sum_outgoing(self, operation):
+        return reduce(operator.add, self.outgoing_edges(operation), StorageEdge(operation))
+    
+    def sum_all_incoming(self):
+        return reduce(operator.add, self.incoming, StorageEdge())
+    
+    def sum_all_outgoing(self):
+        return reduce(operator.add, self.outgoing, StorageEdge())
+    
+    def __str__(self):
+        return self.name
+    
+    def __repr__(self):
+        return "%s(%s)" % (self.__str__(), object.__repr__(self))
+    
+    def merge_edge_sets(self, set1, set2, key_slot):
+        getter = lambda edge: edge.__dict__[key_slot]
+        set_dict = dict([(getter(edge), edge) for edge in set1])
+        for edge in set2:
+            key = getter(edge)
+            if key not in set_dict:
+                set_dict[key] = edge
+            else:
+                set_dict[key] += edge
+        return set(set_dict.values())
+    
+    def __add__(self, other):
+        result = StorageNode("%s %s" % (self.name, other.name))
+        result.incoming = self.merge_edge_sets(self.incoming, other.incoming, "origin")
+        # TODO bad code
+        for edge in result.incoming:
+            edge.target = result
+        result.outgoing = self.merge_edge_sets(self.outgoing, other.outgoing, "target")
+        for edge in result.outgoing:
+            edge.origin = result
+        return result
+    
+    def __lt__(self, other):
+        return self.name < other.name
+    
+    def is_artificial(self):
+        for outgoing in self.outgoing:
+            if outgoing.is_storage_source:
+                return True
+        return False
+    
+    def is_storage_node(self):
+        return self.is_artificial() or self.name in STORAGE_NODES
+    
+    def dot_name(self):
+        return self.name.replace(" ", "_")
+    
+class StorageGraph(object):
+    
+    def __init__(self):
+        self.nodes = {}
+        self.edges = {}
+        self.operations = set()
+    
+    def node(self, name):
+        if str(name) == 'None':
+            import pdb; pdb.set_trace()
+        if name not in self.nodes:
+            self.nodes[name] = StorageNode(name)
+        return self.nodes[name]
+    
+    def assert_sanity(self):
+        visited_edges = set()
+        for node in self.nodes.values():
+            for edge in node.incoming:
+                assert edge in self.edges.values(), "Edge not in graph's edges: %s" % edge
+                visited_edges.add(edge)
+                if not edge.target is node:
+                    print "Wrong edge target: %s\nIncoming edge: %s\nIn node: %s" % (edge.target, edge, node)
+                    assert False
+                if not edge in edge.origin.outgoing:
+                    print "Edge not in origin's outgoing: %s\nIncoming edge: %s\nIn node: %s" % (edge.origin.outgoing, edge, node)
+                    assert False
+            for edge in node.outgoing:
+                assert edge in self.edges.values(), "Edge not in graph's edges: %s" % edge
+                visited_edges.add(edge)
+                if not edge.origin is node:
+                    print "Wrong edge origin: %s\nOutgoing edge: %s\nIn node: %s" % (edge.origin, edge, node)
+                    assert False
+                if not edge in edge.target.incoming:
+                    print "Edge not in origin's incoming: %s\nOutgoing edge: %s\nIn node: %s" % (edge.target.incoming, edge, node)
+                    assert False
+        assert len(visited_edges) == len(self.edges.values()), "Not all of graph's edges visited."
+    
+    def add_log_entry(self, log_entry):
+        self.operations.add(log_entry.operation)
+        key = log_entry.full_key()
+        if key not in self.edges:
+            edge = StorageEdge(log_entry.operation, self.node(log_entry.old_storage), self.node(log_entry.new_storage))
+            self.edges[key] = edge
+            edge.notify_nodes()
+        self.edges[key].add_log_entry(log_entry)
+    
+    def collapse_nodes(self, collapsed_nodes, new_name=None):
+        if len(collapsed_nodes) == 0:
+            return
+        for node in collapsed_nodes:
+            del self.nodes[node.name]
+            for edge in node.incoming:
+                del self.edges[edge.full_key()]
+            for edge in node.outgoing:
+                del self.edges[edge.full_key()]
+        new_node = reduce(operator.add, collapsed_nodes)
+        if new_name is not None:
+            new_node.name = new_name
+        self.nodes[new_node.name] = new_node
+        # TODO bad code
+        for node in collapsed_nodes:
+            for edge in node.incoming:
+                edge.origin.outgoing.remove(edge)
+                new_edges = filter(lambda filtered: filtered.origin == edge.origin, new_node.incoming)
+                assert len(new_edges) == 1
+                edge.origin.outgoing.add(new_edges[0])
+            for edge in node.outgoing:
+                edge.target.incoming.remove(edge)
+                new_edges = filter(lambda filtered: filtered.target == edge.target, new_node.outgoing)
+                assert len(new_edges) == 1
+                edge.target.incoming.add(new_edges[0])
+        for edge in new_node.incoming:
+            self.edges[edge.full_key()] = edge
+        for edge in new_node.outgoing:
+            self.edges[edge.full_key()] = edge
+        self.assert_sanity()
+    
+    def collapse_nonstorage_nodes(self, new_name=None):
+        nodes = filter(lambda x: not x.is_storage_node(), self.nodes.values())
+        self.collapse_nodes(nodes, new_name)
+    
+    def sorted_nodes(self):
+        nodes = self.nodes.values()
+        nodes.sort()
+        return nodes
+    
+def make_graph(logfile, flags):
+    graph = StorageGraph()
+    def callback(entry):
+        graph.add_log_entry(entry)
+    parse(logfile, flags, callback)
+    graph.assert_sanity()
+    return graph
+
+# ====================================================================
+# ======== Command - Summarize log content
+# ====================================================================
+
+def command_summarize(logfile, flags):
+    graph = make_graph(logfile, flags)
+    if not flags.allstorage:
+        graph.collapse_nonstorage_nodes()
+    for node in graph.sorted_nodes():
+        node.print_summary(flags, graph.operations)
+
+def StorageNode_print_summary(self, flags, all_operations):
+    print "\n%s:" % self.name
+    sum = StorageEdge()
+    total_incoming = self.sum_all_incoming().total() if flags.percent else None
+    
+    print "\tIncoming:"
+    for operation in all_operations:
+        if flags.detailed:
+            edges = [ (edge.origin.name, edge) for edge in self.incoming_edges(operation) ]
+        else:
+            edges = [ (operation, self.sum_incoming(operation)) ]
+        for edgename, edge in edges:
+            edge.print_with_name("\t\t\t", edgename, total_incoming, flags)
+            sum += edge
+    
+    print "\tOutgoing:"
+    for operation in all_operations:
+        if flags.detailed:
+            edges = [ (edge.target.name, edge) for edge in self.outgoing_edges(operation) ]
+        else:
+            edges = [ (operation, self.sum_outgoing(operation)) ]
+        for edgename, edge in edges:
+            edge.print_with_name("\t\t\t", edgename, total_incoming, flags)
+            sum -= edge
+    
+    sum.print_with_name("\t", "Remaining", total_incoming, flags)
+
+StorageNode.print_summary = StorageNode_print_summary
+
+def StorageEdge_print_with_name(self, prefix, edgename, total_reference, flags):
+    if flags.classes:   
+        print "%s%s:" % (prefix, edgename)
+        prefix += "\t\t"
+        operations = self.classes.classes.items()
+        operations.sort(reverse=True, key=operator.itemgetter(1))
+    else:
+        operations = [ (edgename, self.total()) ]
+    for classname, classops in operations:
+        classops.prefixprint("%s%s: " % (prefix, classname), total_reference)
+    
+StorageEdge.print_with_name = StorageEdge_print_with_name
+
+# ====================================================================
+# ======== Command - DOT output
+# ====================================================================
+
+# Output is valid dot code and can be parsed by the graphviz dot utility.
+def command_print_dot(logfile, flags):
+    graph = make_graph(logfile, flags)
+    print "/*"
+    print "Storage Statistics (dot format):"
+    print "================================"
+    print "*/"
+    print dot_string(graph, flags)
+
+def run_dot(logfile, flags, output_type):
+    import subprocess
+    dot = dot_string(make_graph(logfile, flags), flags)
+    command = ["dot", "-T%s" % output_type, "-o%s.%s" % (flags.logfile, output_type)]
+    print "Running:\n%s" % " ".join(command)
+    p = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    output = p.communicate(input=dot)[0]
+    print output
+
+def command_dot(logfile, flags):
+    run_dot(logfile, flags, "jpg")
+def command_dot_ps(logfile, flags):
+    run_dot(logfile, flags, "ps")
+def command_dot_pdf(logfile, flags):
+    run_dot(logfile, flags, "pdf")
+def command_dot_svg(logfile, flags):
+    run_dot(logfile, flags, "svg")
+
+def dot_string(graph, flags):
+    result = "digraph G {"
+    incoming_cache = {}
+    if not flags.allstorage:
+        graph.collapse_nonstorage_nodes("Other")
+    
+    def make_label(edge, prefix="", total_edge=None, slots_per_object=False):
+        object_suffix = " objects"
+        slots_suffix = " slots"
+        if not flags.objects or not flags.slots:
+            object_suffix = slots_suffix = ""
+        if total_edge and flags.percent and total_edge.objects != 0:
+            percent_objects = " (%.1f%%)" % percent(edge.objects, total_edge.objects)
+            percent_slots = " (%.1f%%)" % percent(edge.slots, total_edge.slots)
+        else:
+            percent_objects = percent_slots = ""
+        label = ""
+        if flags.objects:
+            label += "%s%s%s%s
" % (prefix, format(edge.objects, ",.0f"), object_suffix, percent_objects)
+        if flags.slots:
+            label += "%s%s%s%s
" % (prefix, format(edge.slots, ",.0f"), slots_suffix, percent_slots)
+        if slots_per_object and flags.slotsPerObject:
+            label += "%.1f slots/object
" % (float(total.slots) / total.objects)
+        return label
+    
+    for node in graph.nodes.values():
+        incoming = node.sum_all_incoming().total()
+        outgoing = node.sum_all_outgoing().total()
+        remaining = incoming - outgoing
+        if node.is_artificial():
+            incoming_cache[node.name] = outgoing
+            shape = ",shape=box"
+            label = make_label(outgoing)
+        else:
+            incoming_cache[node.name] = incoming
+            shape = ""
+            label = make_label(incoming, "Incoming: ")
+            if remaining.objects != incoming.objects:
+                label += make_label(remaining, "Remaining: ", incoming)
+        result += "%s [label=<%s
%s>%s];" % (node.dot_name(), node.name, label, shape)
+    
+    for edge in graph.edges.values():
+        total = edge.total()
+        incoming = incoming_cache[edge.origin.name]
+        label = make_label(total, "", incoming, slots_per_object=True)
+        target_node = edge.target.dot_name()
+        source_node = edge.origin.dot_name()
+        result += "%s -> %s [label=<%s>];" % (source_node, target_node, label)
+    
+    result += "}"
+    return result
+
+# ====================================================================
+# ======== Other commands
+# ====================================================================
+
+def command_aggregate(logfile, flags):
+    graph = make_graph(logfile, flags)
+    edges = graph.edges.values()
+    edges.sort()
+    for edge in edges:
+        logentries = edge.as_log_entries()
+        logentries.sort()
+        for entry in logentries:
+            print entry
+
+def command_print_entries(logfile, flags):
+    def callback(entry):
+        print entry
+    parse(logfile, flags, callback)
+
+# ====================================================================
+# ======== Main
+# ====================================================================
+
+class Flags(object):
+    
+    def __init__(self, flags):
+        self.flags = {}
+        for name, short in flags:
+            self.__dict__[name] = False
+            self.flags[short] = name
+    
+    def handle(self, arg):
+        if arg in self.flags:
+            self.__dict__[self.flags[arg]] = True
+            return True
+        else:
+            return False
+    
+    def __str__(self):
+        descriptions = [ ("%s (%s)" % description) for description in self.flags.items() ]
+        return "[%s]" % " | ".join(descriptions)
+    
+def usage(flags, commands):
+    print "Arguments: logfile command %s" % flags
+    print "Available commands: %s" % commands
+    exit(1)
+
+def main(argv):
+    flags = Flags([
+        # General
+        ('verbose', '-v'),
+        
+        # All outputs
+        ('percent', '-p'),
+        ('allstorage', '-a'),
+        
+        # Text outputs
+        ('detailed', '-d'),
+        ('classes', '-c'),
+        
+        # dot outputs
+        ('slots', '-s'),
+        ('objects', '-o'),
+        ('slotsPerObject', '-S'),
+    ])
+    
+    command_prefix = "command_"
+    module = sys.modules[__name__].__dict__
+    commands = [ a[len(command_prefix):] for a in module.keys() if a.startswith(command_prefix) ]
+    
+    if len(argv) < 2:
+        usage(flags, commands)
+    logfile = argv[0]
+    flags.logfile = logfile
+    for vm_name in AVAILABLE_VMS:
+        if vm_name in logfile:
+            print "Using VM configuration %s" % vm_name
+            SET_VM(vm_name)
+            break
+    command = argv[1]
+    for flag in argv[2:]:
+        if not flags.handle(flag):
+            usage(flags, commands)
+    if command not in commands:
+        usage(flags, commands)
+    
+    func = module[command_prefix + command]
+    func(logfile, flags)
+
+if __name__ == "__main__":
+    main(sys.argv[1:])
diff --git a/rpython/rlib/rstrategies/rstrategies.py b/rpython/rlib/rstrategies/rstrategies.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rstrategies/rstrategies.py
@@ -0,0 +1,570 @@
+
+import weakref, sys
+from rpython.rlib.rstrategies import logger
+from rpython.rlib import jit, objectmodel, rerased
+from rpython.rlib.objectmodel import specialize
+
+def make_accessors(strategy='strategy', storage='storage'):
+    """
+    Instead of using this generator, the methods can be implemented manually.
+    A third way is to overwrite the getter/setter methods in StrategyFactory.
+    """
+    def make_getter(attr):
+        def getter(self): return getattr(self, attr)
+        return getter
+    def make_setter(attr):
+        def setter(self, val): setattr(self, attr, val)
+        return setter
+    classdef = sys._getframe(1).f_locals
+    classdef['_get_strategy'] = make_getter(strategy)
+    classdef['_set_strategy'] = make_setter(strategy)
+    classdef['_get_storage'] = make_getter(storage)
+    classdef['_set_storage'] = make_setter(storage)
+
+class StrategyMetaclass(type):
+    """
+    A metaclass is required, because we need certain attributes to be special
+    for every single strategy class.
+    """
+    def __new__(self, name, bases, attrs):
+        attrs['_is_strategy'] = False
+        attrs['_is_singleton'] = False
+        attrs['_specializations'] = []
+        # Not every strategy uses rerased-pairs, but they won't hurt
+        erase, unerase = rerased.new_erasing_pair(name)
+        def get_storage(self, w_self):
+            erased = self.strategy_factory().get_storage(w_self)
+            return unerase(erased)
+        def set_storage(self, w_self, storage):
+            erased = erase(storage)
+            self.strategy_factory().set_storage(w_self, erased)
+        attrs['get_storage'] = get_storage
+        attrs['set_storage'] = set_storage
+        return type.__new__(self, name, bases, attrs)
+    
+def strategy(generalize=None, singleton=True):
+    """
+    Strategy classes must be decorated with this.
+    generalize is a list of other strategies, that can be switched to from the decorated strategy.
+    If the singleton flag is set to False, new strategy instances will be created,
+    instead of always reusing the singleton object.
+    """
+    def decorator(strategy_class):
+        # Patch strategy class: Add generalized_strategy_for and mark as strategy class.
+        if generalize:
+            @jit.unroll_safe
+            def generalized_strategy_for(self, value):
+                # TODO - optimize this method
+                for strategy in generalize:
+                    if self.strategy_factory().strategy_singleton_instance(strategy).check_can_handle(value):
+                        return strategy
+                raise Exception("Could not find generalized strategy for %s coming from %s" % (value, self))
+            strategy_class.generalized_strategy_for = generalized_strategy_for
+            for generalized in generalize:
+                generalized._specializations.append(strategy_class)
+        strategy_class._is_strategy = True
+        strategy_class._generalizations = generalize
+        strategy_class._is_singleton = singleton
+        return strategy_class
+    return decorator
+
+class StrategyFactory(object):
+    _immutable_fields_ = ["strategies[*]", "logger", "strategy_singleton_field"]
+    factory_instance_counter = 0
+    
+    def __init__(self, root_class, all_strategy_classes=None):
+        if all_strategy_classes is None:
+            all_strategy_classes = self.collect_subclasses(root_class)
+        self.strategies = []
+        self.logger = logger.Logger()
+        
+        # This is to avoid confusion between multiple factories existing simultaneously (e.g. in tests)
+        self.strategy_singleton_field = "__singleton_%i" % StrategyFactory.factory_instance_counter
+        StrategyFactory.factory_instance_counter += 1
+        
+        self.create_strategy_instances(root_class, all_strategy_classes)
+    
+    def create_strategy_instances(self, root_class, all_strategy_classes):
+        for strategy_class in all_strategy_classes:
+            if strategy_class._is_strategy:
+                setattr(strategy_class, self.strategy_singleton_field, self.instantiate_strategy(strategy_class))
+                self.strategies.append(strategy_class)
+            self.patch_strategy_class(strategy_class, root_class)
+        self.order_strategies()
+    
+    # =============================
+    # API methods
+    # =============================
+    
+    def switch_strategy(self, w_self, new_strategy_type, new_element=None):
+        """
+        Switch the strategy of w_self to the new type.
+        new_element can be given as as hint, purely for logging purposes.
+        It should be the object that was added to w_self, causing the strategy switch.
+        """
+        old_strategy = self.get_strategy(w_self)
+        if new_strategy_type._is_singleton:
+            new_strategy = self.strategy_singleton_instance(new_strategy_type)
+        else:
+            size = old_strategy.size(w_self)
+            new_strategy = self.instantiate_strategy(new_strategy_type, w_self, size)
+        self.set_strategy(w_self, new_strategy)
+        old_strategy.convert_storage_to(w_self, new_strategy)
+        new_strategy.strategy_switched(w_self)
+        self.log(w_self, new_strategy, old_strategy, new_element)
+        return new_strategy
+    
+    def set_initial_strategy(self, w_self, strategy_type, size, elements=None):
+        """
+        Initialize the strategy and storage fields of w_self.
+        This must be called before switch_strategy or any strategy method can be used.
+        elements is an optional list of values initially stored in w_self.
+        If given, then len(elements) == size must hold.
+        """
+        assert self.get_strategy(w_self) is None, "Strategy should not be initialized yet!"
+        if strategy_type._is_singleton:
+            strategy = self.strategy_singleton_instance(strategy_type)
+        else:
+            strategy = self.instantiate_strategy(strategy_type, w_self, size)
+        self.set_strategy(w_self, strategy)
+        strategy.initialize_storage(w_self, size)
+        element = None
+        if elements:
+            strategy.store_all(w_self, elements)
+            if len(elements) > 0: element = elements[0]
+        strategy.strategy_switched(w_self)
+        self.log(w_self, strategy, None, element)
+        return strategy
+    
+    @jit.unroll_safe
+    def strategy_type_for(self, objects):
+        """
+        Return the best-fitting strategy to hold all given objects.
+        """
+        specialized_strategies = len(self.strategies)
+        can_handle = [True] * specialized_strategies
+        for obj in objects:
+            if specialized_strategies <= 1:
+                break
+            for i, strategy in enumerate(self.strategies):
+                if can_handle[i] and not self.strategy_singleton_instance(strategy).check_can_handle(obj):
+                    can_handle[i] = False
+                    specialized_strategies -= 1
+        for i, strategy_type in enumerate(self.strategies):
+            if can_handle[i]:
+                return strategy_type
+        raise Exception("Could not find strategy to handle: %s" % objects)
+    
+    def decorate_strategies(self, transitions):
+        """
+        As an alternative to decorating all strategies with @strategy,
+        invoke this in the constructor of your StrategyFactory subclass, before
+        calling __init__. transitions is a dict mapping all strategy classes to
+        their 'generalize' list parameter (see @strategy decorator).
+        """
+        "NOT_RPYTHON"
+        for strategy_class, generalized in transitions.items():
+            strategy(generalized)(strategy_class)
+    
+    # =============================
+    # The following methods can be overwritten to customize certain aspects of the factory.
+    # =============================
+    
+    def instantiate_strategy(self, strategy_type, w_self=None, initial_size=0):
+        """
+        Return a functional instance of strategy_type.
+        Overwrite this if you need a non-default constructor.
+        The two additional parameters should be ignored for singleton-strategies.
+        """
+        return strategy_type()
+    
+    def log(self, w_self, new_strategy, old_strategy=None, new_element=None):
+        """
+        This can be overwritten into a more appropriate call to self.logger.log
+        """
+        if not self.logger.active: return
+        new_strategy_str = self.log_string_for_object(new_strategy)
+        old_strategy_str = self.log_string_for_object(old_strategy)
+        element_typename = self.log_string_for_object(new_element)
+        size = new_strategy.size(w_self)
+        typename = ""
+        cause = "Switched" if old_strategy else "Created"
+        self.logger.log(new_strategy_str, size, cause, old_strategy_str, typename, element_typename)
+    
+    @specialize.call_location()
+    def log_string_for_object(self, obj):
+        """
+        This can be overwritten instead of the entire log() method.
+        Keep the specialize-annotation in order to handle different kinds of objects here.
+        """
+        return obj.__class__.__name__ if obj else ""
+    
+    # These storage accessors are specialized because the storage field is 
+    # populated by erased-objects which seem to be incompatible sometimes.
+    @specialize.call_location()
+    def get_storage(self, obj):
+        return obj._get_storage()
+    @specialize.call_location()
+    def set_storage(self, obj, val):
+        return obj._set_storage(val)
+    
+    def get_strategy(self, obj):
+        return obj._get_strategy()
+    def set_strategy(self, obj, val):
+        return obj._set_strategy(val)
+    
+    # =============================
+    # Internal methods
+    # =============================
+    
+    def patch_strategy_class(self, strategy_class, root_class):
+        "NOT_RPYTHON"
+        # Patch root class: Add default handler for visitor
+        def convert_storage_from_OTHER(self, w_self, previous_strategy):
+            self.convert_storage_from(w_self, previous_strategy)
+        funcname = "convert_storage_from_" + strategy_class.__name__
+        convert_storage_from_OTHER.func_name = funcname
+        setattr(root_class, funcname, convert_storage_from_OTHER)
+        
+        # Patch strategy class: Add polymorphic visitor function
+        def convert_storage_to(self, w_self, new_strategy):
+            getattr(new_strategy, funcname)(w_self, self)
+        strategy_class.convert_storage_to = convert_storage_to
+    
+    def collect_subclasses(self, cls):
+        "NOT_RPYTHON"
+        subclasses = []
+        for subcls in cls.__subclasses__():
+            subclasses.append(subcls)
+            subclasses.extend(self.collect_subclasses(subcls))
+        return subclasses
+    
+    def order_strategies(self):
+        "NOT_RPYTHON"
+        def get_generalization_depth(strategy, visited=None):
+            if visited is None:
+                visited = set()
+            if strategy._generalizations:
+                if strategy in visited:
+                    raise Exception("Cycle in generalization-tree of %s" % strategy)
+                visited.add(strategy)
+                depth = 0
+                for generalization in strategy._generalizations:
+                    other_depth = get_generalization_depth(generalization, visited)
+                    depth = max(depth, other_depth)
+                return depth + 1
+            else:
+                return 0
+        self.strategies.sort(key=get_generalization_depth, reverse=True)
+    
+    @jit.elidable
+    def strategy_singleton_instance(self, strategy_class):
+        return getattr(strategy_class, self.strategy_singleton_field)
+    
+    def _freeze_(self):
+        # Instance will be frozen at compile time, making accesses constant.
+        # The constructor does meta stuff which is not possible after translation.
+        return True
+
+class AbstractStrategy(object):
+    """
+    == Required:
+    strategy_factory(self) - Access to StorageFactory
+    """
+    
+    def strategy_switched(self, w_self):
+        # Overwrite this method for a hook whenever the strategy
+        # of w_self was switched to self.
+        pass
+    
+    # Main Fixedsize API
+    
+    def store(self, w_self, index0, value):
+        raise NotImplementedError("Abstract method")
+    
+    def fetch(self, w_self, index0):
+        raise NotImplementedError("Abstract method")
+    
+    def size(self, w_self):
+        raise NotImplementedError("Abstract method")
+    
+    # Fixedsize utility methods
+    
+    def slice(self, w_self, start, end):
+        return [ self.fetch(w_self, i) for i in range(start, end)]
+    
+    def fetch_all(self, w_self):
+        return self.slice(w_self, 0, self.size(w_self))
+    
+    def store_all(self, w_self, elements):
+        for i, e in enumerate(elements):
+            self.store(w_self, i, e)
+    
+    # Main Varsize API
+    
+    def insert(self, w_self, index0, list_w):
+        raise NotImplementedError("Abstract method")
+    
+    def delete(self, w_self, start, end):
+        raise NotImplementedError("Abstract method")
+    
+    # Varsize utility methods
+    
+    def append(self, w_self, list_w):
+        self.insert(w_self, self.size(w_self), list_w)        
+    
+    def pop(self, w_self, index0):
+        e = self.fetch(w_self, index0)
+        self.delete(w_self, index0, index0+1)
+        return e
+
+    # Internal methods
+    
+    def initialize_storage(self, w_self, initial_size):
+        raise NotImplementedError("Abstract method")
+    
+    def check_can_handle(self, value):
+        raise NotImplementedError("Abstract method")
+    
+    def convert_storage_to(self, w_self, new_strategy):
+        # This will be overwritten in patch_strategy_class
+        new_strategy.convert_storage_from(w_self, self)
+    
+    @jit.unroll_safe
+    def convert_storage_from(self, w_self, previous_strategy):
+        # This is a very unefficient (but most generic) way to do this.
+        # Subclasses should specialize.
+        storage = previous_strategy.fetch_all(w_self)
+        self.initialize_storage(w_self, previous_strategy.size(w_self))
+        for i, field in enumerate(storage):
+            self.store(w_self, i, field)
+    
+    def generalize_for_value(self, w_self, value):
+        strategy_type = self.generalized_strategy_for(value)
+        new_instance = self.strategy_factory().switch_strategy(w_self, strategy_type, new_element=value)
+        return new_instance
+        
+    def cannot_handle_store(self, w_self, index0, value):
+        new_instance = self.generalize_for_value(w_self, value)
+        new_instance.store(w_self, index0, value)
+        
+    def cannot_handle_insert(self, w_self, index0, list_w):
+        # TODO - optimize. Prevent multiple generalizations and slicing done by callers.
+        new_strategy = self.generalize_for_value(w_self, list_w[0])
+        new_strategy.insert(w_self, index0, list_w)
+
+# ============== Special Strategies with no storage array ==============
+
+class EmptyStrategy(AbstractStrategy):
+    # == Required:
+    # See AbstractStrategy
+    
+    def initialize_storage(self, w_self, initial_size):
+        assert initial_size == 0
+        self.set_storage(w_self, None)
+    def convert_storage_from(self, w_self, previous_strategy):
+        self.set_storage(w_self, None)
+    def fetch(self, w_self, index0):
+        raise IndexError
+    def store(self, w_self, index0, value):
+        self.cannot_handle_insert(w_self, index0, [value])
+    def insert(self, w_self, index0, list_w):
+        self.cannot_handle_insert(w_self, index0, list_w)
+    def delete(self, w_self, start, end):
+        self.check_index_range(w_self, start, end)
+    def size(self, w_self):
+        return 0
+    def check_can_handle(self, value):
+        return False
+
+class SingleValueStrategyStorage(object):
+    """Small container object for a size value."""
+    _attrs_ = ['size']
+    def __init__(self, size=0):
+        self.size = size
+
+class SingleValueStrategy(AbstractStrategy):
+    # == Required:
+    # See AbstractStrategy
+    # check_index_*(...) - use mixin SafeIndexingMixin or UnsafeIndexingMixin
+    # value(self) - the single value contained in this strategy. Should be constant.
+    
+    def initialize_storage(self, w_self, initial_size):
+        storage_obj = SingleValueStrategyStorage(initial_size)
+        self.set_storage(w_self, storage_obj)
+    def convert_storage_from(self, w_self, previous_strategy):
+        self.initialize_storage(w_self, previous_strategy.size(w_self))
+    
+    def fetch(self, w_self, index0):
+        self.check_index_fetch(w_self, index0)
+        return self.value()
+    def store(self, w_self, index0, value):
+        self.check_index_store(w_self, index0)
+        if self.check_can_handle(value):
+            return
+        self.cannot_handle_store(w_self, index0, value)
+    
+    @jit.unroll_safe
+    def insert(self, w_self, index0, list_w):
+        storage_obj = self.get_storage(w_self)
+        for i in range(len(list_w)):
+            if self.check_can_handle(list_w[i]):
+                storage_obj.size += 1
+            else:
+                self.cannot_handle_insert(w_self, index0 + i, list_w[i:])
+                return
+    
+    def delete(self, w_self, start, end):
+        self.check_index_range(w_self, start, end)
+        self.get_storage(w_self).size -= (end - start)
+    def size(self, w_self):
+        return self.get_storage(w_self).size
+    def check_can_handle(self, value):
+        return value is self.value()
+    
+# ============== Basic strategies with storage ==============
+
+class StrategyWithStorage(AbstractStrategy):
+    # == Required:
+    # See AbstractStrategy
+    # check_index_*(...) - use mixin SafeIndexingMixin or UnsafeIndexingMixin
+    # default_value(self) - The value to be initially contained in this strategy
+    
+    def initialize_storage(self, w_self, initial_size):
+        default = self._unwrap(self.default_value())
+        self.set_storage(w_self, [default] * initial_size)
+    
+    @jit.unroll_safe
+    def convert_storage_from(self, w_self, previous_strategy):
+        size = previous_strategy.size(w_self)
+        new_storage = [ self._unwrap(previous_strategy.fetch(w_self, i))
+                        for i in range(size) ]
+        self.set_storage(w_self, new_storage)
+    
+    def store(self, w_self, index0, wrapped_value):
+        self.check_index_store(w_self, index0)
+        if self.check_can_handle(wrapped_value):
+            unwrapped = self._unwrap(wrapped_value)
+            self.get_storage(w_self)[index0] = unwrapped
+        else:
+            self.cannot_handle_store(w_self, index0, wrapped_value)
+    
+    def fetch(self, w_self, index0):
+        self.check_index_fetch(w_self, index0)
+        unwrapped = self.get_storage(w_self)[index0]
+        return self._wrap(unwrapped)
+    
+    def _wrap(self, value):
+        raise NotImplementedError("Abstract method")
+    
+    def _unwrap(self, value):
+        raise NotImplementedError("Abstract method")
+    
+    def size(self, w_self):
+        return len(self.get_storage(w_self))
+    
+    @jit.unroll_safe
+    def insert(self, w_self, start, list_w):
+        if start > self.size(w_self):
+            start = self.size(w_self)
+        for i in range(len(list_w)):
+            if self.check_can_handle(list_w[i]):
+                self.get_storage(w_self).insert(start + i, self._unwrap(list_w[i]))
+            else:
+                self.cannot_handle_insert(w_self, start + i, list_w[i:])
+                return
+    
+    def delete(self, w_self, start, end):
+        self.check_index_range(w_self, start, end)
+        assert start >= 0 and end >= 0
+        del self.get_storage(w_self)[start : end]
+        
+class GenericStrategy(StrategyWithStorage):
+    # == Required:
+    # See StrategyWithStorage
+    
+    def _wrap(self, value):
+        return value
+    def _unwrap(self, value):
+        return value
+    def check_can_handle(self, wrapped_value):
+        return True
+    
+class WeakGenericStrategy(StrategyWithStorage):
+    # == Required:
+    # See StrategyWithStorage
+    
+    def _wrap(self, value):
+        return value() or self.default_value()
+    def _unwrap(self, value):
+        assert value is not None
+        return weakref.ref(value)
+    def check_can_handle(self, wrapped_value):
+        return True
+    
+# ============== Mixins for index checking operations ==============
+
+class SafeIndexingMixin(object):
+    def check_index_store(self, w_self, index0):
+        self.check_index(w_self, index0)
+    def check_index_fetch(self, w_self, index0):
+        self.check_index(w_self, index0)
+    def check_index_range(self, w_self, start, end):
+        if end < start:
+            raise IndexError
+        self.check_index(w_self, start)
+        self.check_index(w_self, end)
+    def check_index(self, w_self, index0):
+        if index0 < 0 or index0 >= self.size(w_self):
+            raise IndexError
+
+class UnsafeIndexingMixin(object):
+    def check_index_store(self, w_self, index0):
+        pass
+    def check_index_fetch(self, w_self, index0):
+        pass
+    def check_index_range(self, w_self, start, end):
+        pass
+
+# ============== Specialized Storage Strategies ==============
+
+class SpecializedStrategy(StrategyWithStorage):
+    # == Required:
+    # See StrategyWithStorage
+    # wrap(self, value) - Return a boxed object for the primitive value
+    # unwrap(self, value) - Return the unboxed primitive value of value
+    
+    def _unwrap(self, value):
+        return self.unwrap(value)
+    def _wrap(self, value):
+        return self.wrap(value)
+    
+class SingleTypeStrategy(SpecializedStrategy):
+    # == Required Functions:
+    # See SpecializedStrategy
+    # contained_type - The wrapped type that can be stored in this strategy
+    
+    def check_can_handle(self, value):
+        return isinstance(value, self.contained_type)
+    
+class TaggingStrategy(SingleTypeStrategy):
+    """This strategy uses a special tag value to represent a single additional object."""
+    # == Required:
+    # See SingleTypeStrategy
+    # wrapped_tagged_value(self) - The tagged object
+    # unwrapped_tagged_value(self) - The unwrapped tag value representing the tagged object
+    
+    def check_can_handle(self, value):
+        return value is self.wrapped_tagged_value() or \
+                (isinstance(value, self.contained_type) and \
+                self.unwrap(value) != self.unwrapped_tagged_value())
+    
+    def _unwrap(self, value):
+        if value is self.wrapped_tagged_value():
+            return self.unwrapped_tagged_value()
+        return self.unwrap(value)
+    
+    def _wrap(self, value):
+        if value == self.unwrapped_tagged_value():
+            return self.wrapped_tagged_value()
+        return self.wrap(value)
diff --git a/rpython/rlib/rstrategies/test.py b/rpython/rlib/rstrategies/test.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rstrategies/test.py
@@ -0,0 +1,434 @@
+
+import py
+from rpython.rlib.rstrategies import rstrategies as rs
+from rpython.rlib.objectmodel import import_from_mixin
+
+# === Define small model tree
+
+class W_AbstractObject(object):
+    pass
+
+class W_Object(W_AbstractObject):
+    pass
+
+class W_Integer(W_AbstractObject):
+    def __init__(self, value):
+        self.value = value
+    def __eq__(self, other):
+        return isinstance(other, W_Integer) and self.value == other.value
+
+class W_List(W_AbstractObject):
+    rs.make_accessors()
+    def __init__(self, strategy=None, size=0, elements=None):
+        self.strategy = None
+        if strategy:
+            factory.set_initial_strategy(self, strategy, size, elements)
+    def fetch(self, i):
+        assert self.strategy
+        return self.strategy.fetch(self, i)
+    def store(self, i, value):
+        assert self.strategy
+        return self.strategy.store(self, i, value)
+    def size(self):
+        assert self.strategy
+        return self.strategy.size(self)
+    def insert(self, index0, list_w):
+        assert self.strategy
+        return self.strategy.insert(self, index0, list_w)
+    def delete(self, start, end):
+        assert self.strategy
+        return self.strategy.delete(self, start, end)
+    def append(self, list_w):
+        assert self.strategy
+        return self.strategy.append(self, list_w)
+    def pop(self, index0):
+        assert self.strategy
+        return self.strategy.pop(self, index0)
+    def slice(self, start, end):
+        assert self.strategy
+        return self.strategy.slice(self, start, end)
+    def fetch_all(self):
+        assert self.strategy
+        return self.strategy.fetch_all(self)
+    def store_all(self, elements):
+        assert self.strategy
+        return self.strategy.store_all(self, elements)
+
+w_nil = W_Object()
+
+# === Define concrete strategy classes
+
+class AbstractStrategy(object):
+    __metaclass__ = rs.StrategyMetaclass
+    import_from_mixin(rs.AbstractStrategy)
+    import_from_mixin(rs.SafeIndexingMixin)
+    def __init__(self, factory, w_self=None, size=0):
+        self.factory = factory
+    def strategy_factory(self):
+        return self.factory
+
+class Factory(rs.StrategyFactory):
+    switching_log = []
+    
+    def __init__(self, root_class):
+        self.decorate_strategies({
+            EmptyStrategy: [GenericStrategy],
+            NilStrategy: [IntegerOrNilStrategy, GenericStrategy],
+            GenericStrategy: [],
+            WeakGenericStrategy: [],
+                IntegerStrategy: [IntegerOrNilStrategy, GenericStrategy],
+            IntegerOrNilStrategy: [GenericStrategy],
+        })
+        rs.StrategyFactory.__init__(self, root_class)
+    
+    def instantiate_strategy(self, strategy_type, w_self=None, size=0):
+        return strategy_type(self, w_self, size)
+    
+    def set_strategy(self, w_list, strategy): 
+        old_strategy = self.get_strategy(w_list)
+        self.switching_log.append((old_strategy, strategy))
+        super(Factory, self).set_strategy(w_list, strategy)
+    
+    def clear_log(self):
+        del self.switching_log[:]
+
+class EmptyStrategy(AbstractStrategy):
+    import_from_mixin(rs.EmptyStrategy)
+
+class NilStrategy(AbstractStrategy):
+    import_from_mixin(rs.SingleValueStrategy)
+    def value(self): return w_nil
+
+class GenericStrategy(AbstractStrategy):
+    import_from_mixin(rs.GenericStrategy)
+    import_from_mixin(rs.UnsafeIndexingMixin)
+    def default_value(self): return w_nil
+
+class WeakGenericStrategy(AbstractStrategy):
+    import_from_mixin(rs.WeakGenericStrategy)
+    def default_value(self): return w_nil
+    
+class IntegerStrategy(AbstractStrategy):
+    import_from_mixin(rs.SingleTypeStrategy)
+    contained_type = W_Integer
+    def wrap(self, value): return W_Integer(value)
+    def unwrap(self, value): return value.value
+    def default_value(self): return W_Integer(0)
+
+class IntegerOrNilStrategy(AbstractStrategy):
+    import_from_mixin(rs.TaggingStrategy)
+    contained_type = W_Integer
+    def wrap(self, value): return W_Integer(value)
+    def unwrap(self, value): return value.value
+    def default_value(self): return w_nil
+    def wrapped_tagged_value(self): return w_nil
+    def unwrapped_tagged_value(self): import sys; return sys.maxint
+    
+ at rs.strategy(generalize=[], singleton=False)
+class NonSingletonStrategy(GenericStrategy):
+    def __init__(self, factory, w_list=None, size=0):
+        super(NonSingletonStrategy, self).__init__(factory, w_list, size)
+        self.w_list = w_list
+        self.size = size
+
+class NonStrategy(NonSingletonStrategy):
+    pass
+
+factory = Factory(AbstractStrategy)
+
+def check_contents(list, expected):
+    assert list.size() == len(expected)
+    for i, val in enumerate(expected):
+        assert list.fetch(i) == val
+
+def teardown():
+    factory.clear_log()
+
+# === Test Initialization and fetch
+
+def test_setup():
+    pass
+
+def test_factory_setup():
+    expected_strategies = 7
+    assert len(factory.strategies) == expected_strategies
+    assert len(set(factory.strategies)) == len(factory.strategies)
+    for strategy in factory.strategies:
+        assert isinstance(factory.strategy_singleton_instance(strategy), strategy)
+
+def test_factory_setup_singleton_instances():
+    new_factory = Factory(AbstractStrategy)
+    s1 = factory.strategy_singleton_instance(GenericStrategy)
+    s2 = new_factory.strategy_singleton_instance(GenericStrategy)
+    assert s1 is not s2
+    assert s1.strategy_factory() is factory
+    assert s2.strategy_factory() is new_factory
+
+def test_metaclass():
+    assert NonStrategy._is_strategy == False
+    assert IntegerOrNilStrategy._is_strategy == True
+    assert IntegerOrNilStrategy._is_singleton == True
+    assert NonSingletonStrategy._is_singleton == False
+    assert NonStrategy._is_singleton == False
+    assert NonStrategy.get_storage is not NonSingletonStrategy.get_storage
+
+def test_singletons():
+    def do_test_singletons(cls, expected_true):
+        l1 = W_List(cls, 0)
+        l2 = W_List(cls, 0)
+        if expected_true:
+            assert l1.strategy is l2.strategy
+        else:
+            assert l1.strategy is not l2.strategy
+    do_test_singletons(EmptyStrategy, True)
+    do_test_singletons(NonSingletonStrategy, False)
+    do_test_singletons(NonStrategy, False)
+    do_test_singletons(GenericStrategy, True)
+
+def do_test_initialization(cls, default_value=w_nil, is_safe=True):
+    size = 10
+    l = W_List(cls, size)
+    s = l.strategy
+    assert s.size(l) == size
+    assert s.fetch(l,0) == default_value
+    assert s.fetch(l,size/2) == default_value
+    assert s.fetch(l,size-1) == default_value
+    py.test.raises(IndexError, s.fetch, l, size)
+    py.test.raises(IndexError, s.fetch, l, size+1)
+    py.test.raises(IndexError, s.fetch, l, size+5)
+    if is_safe:
+        py.test.raises(IndexError, s.fetch, l, -1)
+    else:
+        assert s.fetch(l, -1) == s.fetch(l, size - 1)
+
+def test_init_Empty():
+    l = W_List(EmptyStrategy, 0)
+    s = l.strategy
+    assert s.size(l) == 0
+    py.test.raises(IndexError, s.fetch, l, 0)
+    py.test.raises(IndexError, s.fetch, l, 10)
+    
+def test_init_Nil():
+    do_test_initialization(NilStrategy)
+
+def test_init_Generic():
+    do_test_initialization(GenericStrategy, is_safe=False)
+    
+def test_init_WeakGeneric():
+    do_test_initialization(WeakGenericStrategy)
+    
+def test_init_Integer():
+    do_test_initialization(IntegerStrategy, default_value=W_Integer(0))
+    
+def test_init_IntegerOrNil():
+    do_test_initialization(IntegerOrNilStrategy)
+    
+# === Test Simple store
+
+def do_test_store(cls, stored_value=W_Object(), is_safe=True, is_varsize=False):
+    size = 10
+    l = W_List(cls, size)
+    s = l.strategy
+    def store_test(index):
+        s.store(l, index, stored_value)
+        assert s.fetch(l, index) == stored_value
+    store_test(0)
+    store_test(size/2)
+    store_test(size-1)
+    if not is_varsize:
+        py.test.raises(IndexError, s.store, l, size, stored_value)
+        py.test.raises(IndexError, s.store, l, size+1, stored_value)
+        py.test.raises(IndexError, s.store, l, size+5, stored_value)
+    if is_safe:
+        py.test.raises(IndexError, s.store, l, -1, stored_value)
+    else:
+        store_test(-1)
+
+def test_store_Nil():
+    do_test_store(NilStrategy, stored_value=w_nil)
+
+def test_store_Generic():
+    do_test_store(GenericStrategy, is_safe=False)
+    
+def test_store_WeakGeneric():
+    do_test_store(WeakGenericStrategy, stored_value=w_nil)
+    
+def test_store_Integer():
+    do_test_store(IntegerStrategy, stored_value=W_Integer(100))
+    
+def test_store_IntegerOrNil():
+    do_test_store(IntegerOrNilStrategy, stored_value=W_Integer(100))
+    do_test_store(IntegerOrNilStrategy, stored_value=w_nil)
+
+# === Test Insert
+
+def do_test_insert(cls, values):
+    l = W_List(cls, 0)
+    assert len(values) >= 6
+    values1 = values[0:2]
+    values2 = values[2:4]
+    values3 = values[4:6]
+    l.insert(0, values1+values3)
+    check_contents(l, values1+values3)
+    l.insert(2, values2)
+    check_contents(l, values)
+
+def test_insert_Nil():
+    do_test_insert(NilStrategy, [w_nil]*6)
+
+def test_insert_Generic():
+    do_test_insert(GenericStrategy, [W_Object() for _ in range(6)])
+    
+def test_insert_WeakGeneric():
+    do_test_insert(WeakGenericStrategy, [W_Object() for _ in range(6)])
+    
+def test_insert_Integer():
+    do_test_insert(IntegerStrategy, [W_Integer(x) for x in range(6)])
+    
+def test_insert_IntegerOrNil():
+    do_test_insert(IntegerOrNilStrategy, [w_nil]+[W_Integer(x) for x in range(4)]+[w_nil])
+    do_test_insert(IntegerOrNilStrategy, [w_nil]*6)
+    
+# === Test Delete
+
+def do_test_delete(cls, values):
+    assert len(values) >= 6
+    l = W_List(cls, len(values), values)
+    l.delete(2, 4)
+    del values[2: 4]
+    check_contents(l, values)
+    l.delete(1, 2)
+    del values[1: 2]
+    check_contents(l, values)
+
+def test_delete_Nil():
+    do_test_delete(NilStrategy, [w_nil]*6)
+
+def test_delete_Generic():
+    do_test_delete(GenericStrategy, [W_Object() for _ in range(6)])
+    
+def test_delete_WeakGeneric():
+    do_test_delete(WeakGenericStrategy, [W_Object() for _ in range(6)])
+    
+def test_delete_Integer():
+    do_test_delete(IntegerStrategy, [W_Integer(x) for x in range(6)])
+    
+def test_delete_IntegerOrNil():
+    do_test_delete(IntegerOrNilStrategy, [w_nil]+[W_Integer(x) for x in range(4)]+[w_nil])
+    do_test_delete(IntegerOrNilStrategy, [w_nil]*6)
+
+# === Test Transitions
+
+def test_CheckCanHandle():
+    def assert_handles(cls, good, bad):
+        s = cls(0)
+        for val in good:
+            assert s.check_can_handle(val)
+        for val in bad:
+            assert not s.check_can_handle(val)
+    obj = W_Object()
+    i = W_Integer(0)
+    nil = w_nil
+    
+    assert_handles(EmptyStrategy, [], [nil, obj, i])
+    assert_handles(NilStrategy, [nil], [obj, i])
+    assert_handles(GenericStrategy, [nil, obj, i], [])
+    assert_handles(WeakGenericStrategy, [nil, obj, i], [])
+    assert_handles(IntegerStrategy, [i], [nil, obj])
+    assert_handles(IntegerOrNilStrategy, [nil, i], [obj])
+
+def do_test_transition(OldStrategy, value, NewStrategy, initial_size=10):
+    w = W_List(OldStrategy, initial_size)
+    old = w.strategy
+    w.store(0, value)
+    assert isinstance(w.strategy, NewStrategy)
+    assert factory.switching_log == [(None, old), (old, w.strategy)]
+
+def test_AllNil_to_Generic():
+    do_test_transition(NilStrategy, W_Object(), GenericStrategy)
+
+def test_AllNil_to_IntegerOrNil():
+    do_test_transition(NilStrategy, W_Integer(0), IntegerOrNilStrategy)
+
+def test_IntegerOrNil_to_Generic():
+    do_test_transition(IntegerOrNilStrategy, W_Object(), GenericStrategy)
+
+def test_Integer_to_IntegerOrNil():
+    do_test_transition(IntegerStrategy, w_nil, IntegerOrNilStrategy)
+
+def test_Integer_Generic():
+    do_test_transition(IntegerStrategy, W_Object(), GenericStrategy)
+
+def test_TaggingValue_not_storable():
+    tag = IntegerOrNilStrategy(10).unwrapped_tagged_value() # sys.maxint
+    do_test_transition(IntegerOrNilStrategy, W_Integer(tag), GenericStrategy)
+
+# TODO - Test transition from varsize back to Empty
+
+# === Test helper methods
+
+def generic_list():
+    values = [W_Object() for _ in range(6)]
+    return W_List(GenericStrategy, len(values), values), values
+
+def test_slice():
+    l, v = generic_list()
+    assert l.slice(2, 4) == v[2:4]
+
+def test_fetch_all():
+    l, v = generic_list()
+    assert l.fetch_all() == v
+
+def test_append():
+    l, v = generic_list()
+    o1 = W_Object()
+    o2 = W_Object()
+    l.append([o1])
+    assert l.fetch_all() == v + [o1]
+    l.append([o1, o2])
+    assert l.fetch_all() == v + [o1, o1, o2]
+
+def test_pop():
+    l, v = generic_list()
+    o = l.pop(3)
+    del v[3]
+    assert l.fetch_all() == v
+    o = l.pop(3)
+    del v[3]
+    assert l.fetch_all() == v
+
+def test_store_all():
+    l, v = generic_list()
+    v2 = [W_Object() for _ in range(4) ]
+    v3 = [W_Object() for _ in range(l.size()) ]
+    assert v2 != v
+    assert v3 != v
+    
+    l.store_all(v2)
+    assert l.fetch_all() == v2+v[4:]
+    l.store_all(v3)
+    assert l.fetch_all() == v3
+    
+    py.test.raises(IndexError, l.store_all, [W_Object() for _ in range(8) ])
+
+# === Test Weak Strategy
+# TODO
+
+# === Other tests
+
+def test_optimized_strategy_switch(monkeypatch):
+    l = W_List(NilStrategy, 5)
+    s = l.strategy
+    s.copied = 0
+    def convert_storage_from_default(self, w_self, other):
+        assert False, "The default convert_storage_from() should not be called!"
+    def convert_storage_from_special(self, w_self, other):
+        s.copied += 1
+    
+    monkeypatch.setattr(AbstractStrategy, "convert_storage_from_NilStrategy", convert_storage_from_special)
+    monkeypatch.setattr(AbstractStrategy, "convert_storage_from", convert_storage_from_default)
+    try:
+        factory.switch_strategy(l, IntegerOrNilStrategy)
+    finally:
+        monkeypatch.undo()
+    assert s.copied == 1, "Optimized switching routine not called exactly one time."

From noreply at buildbot.pypy.org  Wed Apr 22 12:20:43 2015
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Wed, 22 Apr 2015 12:20:43 +0200 (CEST)
Subject: [pypy-commit] pypy default: Merged pypy/pypy into default
Message-ID: <20150422102043.6D6121C06CD@cobra.cs.uni-duesseldorf.de>

Author: anton_gulenko 
Branch: 
Changeset: r76869:9ca81bee353c
Date: 2015-02-22 11:45 +0100
http://bitbucket.org/pypy/pypy/changeset/9ca81bee353c/

Log:	Merged pypy/pypy into default

diff too long, truncating to 2000 out of 2663 lines

diff --git a/lib_pypy/audioop.py b/lib_pypy/audioop.py
--- a/lib_pypy/audioop.py
+++ b/lib_pypy/audioop.py
@@ -480,6 +480,7 @@
                             state_d, prev_i, cur_i,
                             weightA, weightB)
     result = ffi.buffer(rv)[:trim_index]
+    d = state_d[0]
     samps = zip(prev_i, cur_i)
     return (result, (d, tuple(samps)))
 
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -6,8 +6,8 @@
 .. startrev: 397b96217b85
 
 
-Fix non-blocking file reads sometimes raising EAGAIN even though they
-have buffered data waiting (b1c4fcb04a42)
+Non-blocking file reads sometimes raised EAGAIN even though they
+had buffered data waiting, fixed in b1c4fcb04a42
 
 
 .. branch: vmprof
@@ -18,3 +18,15 @@
 
 .. branch: stdlib-2.7.9
 Update stdlib to version 2.7.9
+
+.. branch: fix-kqueue-error2
+Fix exception being raised by kqueue.control (CPython compatibility)
+
+.. branch: gitignore
+
+.. branch: framestate2
+Refactor rpython.flowspace.framestate.FrameState.
+
+.. branch: alt_errno
+Add an alternative location to save LastError, errno around ctypes,
+cffi external calls so things like pdb will not overwrite it
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -25,8 +25,8 @@
         self.reason = reason
 
 @specialize.memo()
-def encode_error_handler(space):
-    # Fast version of the "strict" errors handler.
+def rpy_encode_error_handler():
+    # A RPython version of the "strict" error handler.
     def raise_unicode_exception_encode(errors, encoding, msg, u,
                                        startingpos, endingpos):
         raise RUnicodeEncodeError(encoding, u, startingpos, endingpos, msg)
@@ -62,7 +62,10 @@
     return result
 
 def encode_utf8(space, uni):
+    # Note that this function never raises UnicodeEncodeError,
+    # since surrogate pairs are allowed.
+    # This is not the case with Python3.
     return runicode.unicode_encode_utf_8(
         uni, len(uni), "strict",
-        errorhandler=encode_error_handler(space),
+        errorhandler=rpy_encode_error_handler(),
         allow_surrogates=True)
diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py
--- a/pypy/module/_cffi_backend/ccallback.py
+++ b/pypy/module/_cffi_backend/ccallback.py
@@ -210,6 +210,6 @@
         space.threadlocals.leave_thread(space)
 
 def invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata):
-    cerrno._errno_after(rffi.RFFI_ERR_ALL)
+    cerrno._errno_after(rffi.RFFI_ERR_ALL | rffi.RFFI_ALT_ERRNO)
     _invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata)
-    cerrno._errno_before(rffi.RFFI_ERR_ALL)
+    cerrno._errno_before(rffi.RFFI_ERR_ALL | rffi.RFFI_ALT_ERRNO)
diff --git a/pypy/module/_cffi_backend/cerrno.py b/pypy/module/_cffi_backend/cerrno.py
--- a/pypy/module/_cffi_backend/cerrno.py
+++ b/pypy/module/_cffi_backend/cerrno.py
@@ -13,18 +13,18 @@
 _errno_after  = rposix._errno_after
 
 def get_errno(space):
-    return space.wrap(rposix.get_saved_errno())
+    return space.wrap(rposix.get_saved_alterrno())
 
 @unwrap_spec(errno=int)
 def set_errno(space, errno):
-    rposix.set_saved_errno(errno)
+    rposix.set_saved_alterrno(errno)
 
 # ____________________________________________________________
 
 @unwrap_spec(code=int)
 def getwinerror(space, code=-1):
-    from rpython.rlib.rwin32 import GetLastError_saved, FormatError
+    from rpython.rlib.rwin32 import GetLastError_alt_saved, FormatError
     if code == -1:
-        code = GetLastError_saved()
+        code = GetLastError_alt_saved()
     message = FormatError(code)
     return space.newtuple([space.wrap(code), space.wrap(message)])
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -2716,6 +2716,14 @@
     assert data == b"Xhello\n"
     posix.close(fdr)
 
+def test_errno_saved():
+    set_errno(42)
+    # a random function that will reset errno to 0 (at least on non-windows)
+    import os; os.stat('.')
+    #
+    res = get_errno()
+    assert res == 42
+
 def test_GetLastError():
     if sys.platform != "win32":
         py.test.skip("GetLastError(): only for Windows")
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -608,19 +608,19 @@
     return space.wrap(W_CDLL(space, name, cdll))
 
 def get_errno(space):
-    return space.wrap(rposix.get_saved_errno())
+    return space.wrap(rposix.get_saved_alterrno())
 
 def set_errno(space, w_errno):
-    rposix.set_saved_errno(space.int_w(w_errno))
+    rposix.set_saved_alterrno(space.int_w(w_errno))
 
 if sys.platform == 'win32':
     # see also
     # https://bitbucket.org/pypy/pypy/issue/1944/ctypes-on-windows-getlasterror
     def get_last_error(space):
-        return space.wrap(rwin32.GetLastError_saved())
+        return space.wrap(rwin32.GetLastError_alt_saved())
     @unwrap_spec(error=int)
     def set_last_error(space, error):
-        rwin32.SetLastError_saved(error)
+        rwin32.SetLastError_alt_saved(error)
 else:
     # always have at least a dummy version of these functions
     # (https://bugs.pypy.org/issue1242)
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -1,6 +1,8 @@
 from rpython.rlib import rpoll, rsocket, rthread, rweakref
 from rpython.rlib.rarithmetic import intmask, widen, r_uint
 from rpython.rlib.ropenssl import *
+from pypy.module._socket import interp_socket
+from rpython.rlib._rsocket_rffi import MAX_FD_SIZE
 from rpython.rlib.rposix import get_saved_errno
 from rpython.rlib.rweakref import RWeakValueDictionary
 from rpython.rlib.objectmodel import specialize, compute_unique_id
@@ -12,7 +14,6 @@
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.module._ssl.ssl_data import (
     LIBRARY_CODES_TO_NAMES, ERROR_CODES_TO_NAMES)
-from pypy.module._socket import interp_socket
 
 
 # user defined constants
@@ -79,7 +80,8 @@
 constants["OP_CIPHER_SERVER_PREFERENCE"] = SSL_OP_CIPHER_SERVER_PREFERENCE
 constants["OP_SINGLE_DH_USE"] = SSL_OP_SINGLE_DH_USE
 constants["OP_SINGLE_ECDH_USE"] = SSL_OP_SINGLE_ECDH_USE
-constants["OP_NO_COMPRESSION"] = SSL_OP_NO_COMPRESSION
+if SSL_OP_NO_COMPRESSION is not None:
+    constants["OP_NO_COMPRESSION"] = SSL_OP_NO_COMPRESSION
 
 constants["OPENSSL_VERSION_NUMBER"] = OPENSSL_VERSION_NUMBER
 ver = OPENSSL_VERSION_NUMBER
@@ -886,9 +888,13 @@
         libssl_AUTHORITY_INFO_ACCESS_free(info)
 
 def _get_crl_dp(space, certificate):
-    # Calls x509v3_cache_extensions and sets up crldp
-    libssl_X509_check_ca(certificate)
-    dps = certificate[0].c_crldp
+    if OPENSSL_VERSION_NUMBER >= 0x10001000:
+        # Calls x509v3_cache_extensions and sets up crldp
+        libssl_X509_check_ca(certificate)
+        dps = certificate[0].c_crldp
+    else:
+        dps = rffi.cast(stack_st_DIST_POINT, libssl_X509_get_ext_d2i(
+            certificate, NID_crl_distribution_points, None, None))
     if not dps:
         return None
 
diff --git a/pypy/module/_ssl/ssl_data.py b/pypy/module/_ssl/ssl_data.py
--- a/pypy/module/_ssl/ssl_data.py
+++ b/pypy/module/_ssl/ssl_data.py
@@ -369,7 +369,8 @@
     LIBRARY_CODES_TO_NAMES[cconfig[code]] = code
 ERROR_CODES_TO_NAMES = {}
 for lib, code in error_codes:
-    ERROR_CODES_TO_NAMES[cconfig[lib], cconfig[code]] = code
+    if cconfig[code] is not None:
+        ERROR_CODES_TO_NAMES[cconfig[lib], cconfig[code]] = code
 
 ALERT_DESCRIPTION_CODES = {}
 for name in AD_NAMES:
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -236,6 +236,9 @@
 
     def test_npn_protocol(self):
         import socket, _ssl, gc
+        if not _ssl.HAS_NPN:
+            skip("NPN requires OpenSSL 1.0.1 or greater")
+
         ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
         ctx._set_npn_protocols(b'\x08http/1.1\x06spdy/2')
         ss = ctx._wrap_socket(self.s._sock, True,
@@ -307,12 +310,13 @@
             os.path.dirname(__file__), 'dh512.pem'))
 
     def test_load_cert_chain(self):
-        import _ssl
+        import _ssl, errno
         ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
         ctx.load_cert_chain(self.keycert)
         ctx.load_cert_chain(self.cert, self.key)
-        raises(IOError, ctx.load_cert_chain, "inexistent.pem")
-        raises(_ssl.SSLError, ctx.load_cert_chain, self.badcert)
+        exc = raises(IOError, ctx.load_cert_chain, "inexistent.pem")
+        assert exc.value.errno == errno.ENOENT
+        exc = raises(_ssl.SSLError, ctx.load_cert_chain, self.badcert)
         raises(_ssl.SSLError, ctx.load_cert_chain, self.emptycert)
         # Password protected key and cert
         raises(_ssl.SSLError, ctx.load_cert_chain, self.cert_protected,
@@ -360,12 +364,14 @@
         assert ctx.cert_store_stats() == {'x509_ca': 0, 'crl': 0, 'x509': 1}
 
     def test_load_dh_params(self):
-        import _ssl
+        import _ssl, errno
         ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
         ctx.load_dh_params(self.dh512)
         raises(TypeError, ctx.load_dh_params)
         raises(TypeError, ctx.load_dh_params, None)
         raises(_ssl.SSLError, ctx.load_dh_params, self.keycert)
+        exc = raises(IOError, ctx.load_dh_params, "inexistent.pem")
+        assert exc.value.errno == errno.ENOENT
 
     def test_set_ecdh_curve(self):
         import _ssl
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,7 +29,7 @@
 #define PY_VERSION		"2.7.9"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "2.6.0"
+#define PYPY_VERSION "2.6.0-alpha0"
 
 /* Subversion Revision number of this file (not of the repository).
  * Empty since Mercurial migration. */
diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -202,7 +202,7 @@
         assert loop.match_by_id('cfficall', """
             p96 = force_token()
             setfield_gc(p0, p96, descr=)
-            f97 = call_release_gil(27, i59, 1.0, 3, descr=)
+            f97 = call_release_gil(91, i59, 1.0, 3, descr=)
             guard_not_forced(descr=...)
             guard_no_exception(descr=...)
         """, ignore_ops=['guard_not_invalidated'])
diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py
--- a/pypy/module/pypyjit/test_pypy_c/test_thread.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py
@@ -67,21 +67,10 @@
         i58 = call_release_gil(0, _, i37, 1, descr=)
         guard_not_forced(descr=...)
         guard_no_exception(descr=...)
-        i59 = int_is_true(i58)
-        guard_true(i59, descr=...)
-        i60 = int_sub(i44, 1)
-        p62 = force_token()
-        setfield_gc(p0, p62, descr=)
-        i63 = call_release_gil(0, _, i37, 0, descr=)
-        guard_not_forced(descr=...)
-        guard_no_exception(descr=...)
-        i64 = int_is_true(i63)
-        guard_false(i64, descr=...)
-        p65 = force_token()
-        setfield_gc(p0, p65, descr=)
-        call_release_gil(0, _, i37, descr=)
-        guard_not_forced(descr=...)
-        guard_no_exception(descr=...)
+        i58 = int_sub(i44, 1)
+        i59 = call(ConstClass(RPyThreadReleaseLock), i37, descr=)
+        i60 = int_is_true(i59)
+        guard_false(i60, descr=...)
         guard_not_invalidated(descr=...)
         --TICK--
         jump(..., descr=...)
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c b/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c
--- a/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c
+++ b/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c
@@ -272,7 +272,11 @@
 {
 	double x, sum=0.0, dx=(b-a)/(double)nstep;
 	for(x=a+0.5*dx; (b-x)*(x-a)>0.0; x+=dx)
+    {   
+        double y = f(x);
+        printf("f(x)=%.1f\n", y);
 		sum += f(x);
+    }
 	return sum/(double)nstep;
 }
 
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py b/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py
@@ -138,6 +138,7 @@
         integrate.restype = c_double
 
         def func(x):
+            print 'calculating x**2 of',x
             return x**2
 
         result = integrate(0.0, 1.0, CALLBACK(func), 10)
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -168,7 +168,6 @@
 if cConfig.has_gettimeofday:
     c_gettimeofday = external('gettimeofday', [rffi.VOIDP, rffi.VOIDP], rffi.INT)
 TM_P = lltype.Ptr(tm)
-c_clock = external('clock', [rffi.TIME_TP], clock_t)
 c_time = external('time', [rffi.TIME_TP], rffi.TIME_T)
 c_ctime = external('ctime', [rffi.TIME_TP], rffi.CCHARP)
 c_gmtime = external('gmtime', [rffi.TIME_TP], TM_P,
diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -427,9 +427,11 @@
 
     def descr_bit_length(self, space):
         val = self.intval
+        bits = 0
         if val < 0:
-            val = -val
-        bits = 0
+            # warning, "-val" overflows here
+            val = -((val + 1) >> 1)
+            bits = 1
         while val:
             bits += 1
             val >>= 1
diff --git a/pypy/objspace/std/test/test_intobject.py b/pypy/objspace/std/test/test_intobject.py
--- a/pypy/objspace/std/test/test_intobject.py
+++ b/pypy/objspace/std/test/test_intobject.py
@@ -521,11 +521,20 @@
             (10, 4),
             (150, 8),
             (-1, 1),
+            (-2, 2),
+            (-3, 2),
+            (-4, 3),
             (-10, 4),
             (-150, 8),
         ]:
             assert val.bit_length() == bits
 
+    def test_bit_length_max(self):
+        import sys
+        val = -sys.maxint-1
+        bits = 32 if val == -2147483648 else 64
+        assert val.bit_length() == bits
+
     def test_int_real(self):
         class A(int): pass
         b = A(5).real
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -439,12 +439,12 @@
             try:
                 if encoding == 'ascii':
                     u = space.unicode_w(w_object)
-                    eh = unicodehelper.encode_error_handler(space)
+                    eh = unicodehelper.rpy_encode_error_handler()
                     return space.wrap(unicode_encode_ascii(
                             u, len(u), None, errorhandler=eh))
                 if encoding == 'utf-8':
                     u = space.unicode_w(w_object)
-                    eh = unicodehelper.encode_error_handler(space)
+                    eh = unicodehelper.rpy_encode_error_handler()
                     return space.wrap(unicode_encode_utf_8(
                             u, len(u), None, errorhandler=eh,
                             allow_surrogates=True))
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -12,8 +12,7 @@
 from rpython.flowspace.argument import CallSpec
 from rpython.flowspace.model import (Constant, Variable, Block, Link,
     c_last_exception, const, FSException)
-from rpython.flowspace.framestate import (FrameState, recursively_unflatten,
-    recursively_flatten)
+from rpython.flowspace.framestate import FrameState
 from rpython.flowspace.specialcase import (rpython_print_item,
     rpython_print_newline)
 from rpython.flowspace.operation import op
@@ -278,6 +277,7 @@
     "cmp_exc_match",
     ]
 
+
 class FlowContext(object):
     def __init__(self, graph, code):
         self.graph = graph
@@ -307,112 +307,91 @@
 
         The locals are ordered according to self.pycode.signature.
         """
-        self.valuestackdepth = code.co_nlocals
-        self.locals_stack_w = [None] * (code.co_stacksize + code.co_nlocals)
+        self.nlocals = code.co_nlocals
+        self.locals_w = [None] * code.co_nlocals
+        self.stack = []
+
+    @property
+    def stackdepth(self):
+        return len(self.stack)
 
     def pushvalue(self, w_object):
-        depth = self.valuestackdepth
-        self.locals_stack_w[depth] = w_object
-        self.valuestackdepth = depth + 1
+        self.stack.append(w_object)
 
     def popvalue(self):
-        depth = self.valuestackdepth - 1
-        assert depth >= self.pycode.co_nlocals, "pop from empty value stack"
-        w_object = self.locals_stack_w[depth]
-        self.locals_stack_w[depth] = None
-        self.valuestackdepth = depth
-        return w_object
+        return self.stack.pop()
 
     def peekvalue(self, index_from_top=0):
         # NOTE: top of the stack is peekvalue(0).
-        index = self.valuestackdepth + ~index_from_top
-        assert index >= self.pycode.co_nlocals, (
-            "peek past the bottom of the stack")
-        return self.locals_stack_w[index]
+        index = ~index_from_top
+        return self.stack[index]
 
     def settopvalue(self, w_object, index_from_top=0):
-        index = self.valuestackdepth + ~index_from_top
-        assert index >= self.pycode.co_nlocals, (
-            "settop past the bottom of the stack")
-        self.locals_stack_w[index] = w_object
+        index = ~index_from_top
+        self.stack[index] = w_object
 
     def popvalues(self, n):
-        values_w = [self.popvalue() for i in range(n)]
-        values_w.reverse()
+        if n == 0:
+            return []
+        values_w = self.stack[-n:]
+        del self.stack[-n:]
         return values_w
 
-    def dropvalues(self, n):
-        finaldepth = self.valuestackdepth - n
-        for n in range(finaldepth, self.valuestackdepth):
-            self.locals_stack_w[n] = None
-        self.valuestackdepth = finaldepth
-
     def dropvaluesuntil(self, finaldepth):
-        for n in range(finaldepth, self.valuestackdepth):
-            self.locals_stack_w[n] = None
-        self.valuestackdepth = finaldepth
-
-    def save_locals_stack(self):
-        return self.locals_stack_w[:self.valuestackdepth]
-
-    def restore_locals_stack(self, items_w):
-        self.locals_stack_w[:len(items_w)] = items_w
-        self.dropvaluesuntil(len(items_w))
+        del self.stack[finaldepth:]
 
     def getstate(self, next_offset):
-        # getfastscope() can return real None, for undefined locals
-        data = self.save_locals_stack()
-        if self.last_exception is None:
-            data.append(Constant(None))
-            data.append(Constant(None))
-        else:
-            data.append(self.last_exception.w_type)
-            data.append(self.last_exception.w_value)
-        recursively_flatten(data)
-        return FrameState(data, self.blockstack[:], next_offset)
+        return FrameState(self.locals_w[:], self.stack[:],
+                self.last_exception, self.blockstack[:], next_offset)
 
     def setstate(self, state):
         """ Reset the context to the given frame state. """
-        data = state.mergeable[:]
-        recursively_unflatten(data)
-        self.restore_locals_stack(data[:-2])  # Nones == undefined locals
-        if data[-2] == Constant(None):
-            assert data[-1] == Constant(None)
-            self.last_exception = None
-        else:
-            self.last_exception = FSException(data[-2], data[-1])
+        self.locals_w = state.locals_w[:]
+        self.stack = state.stack[:]
+        self.last_exception = state.last_exception
         self.blockstack = state.blocklist[:]
+        self._normalize_raise_signals()
+
+    def _normalize_raise_signals(self):
+        st = self.stack
+        for i in range(len(st)):
+            if isinstance(st[i], RaiseImplicit):
+                st[i] = Raise(st[i].w_exc)
 
     def guessbool(self, w_condition):
         if isinstance(w_condition, Constant):
             return w_condition.value
         return self.recorder.guessbool(self, w_condition)
 
-    def record(self, spaceop):
+    def maybe_merge(self):
         recorder = self.recorder
         if getattr(recorder, 'final_state', None) is not None:
             self.mergeblock(recorder.crnt_block, recorder.final_state)
             raise StopFlowing
+
+    def record(self, spaceop):
         spaceop.offset = self.last_offset
-        recorder.append(spaceop)
+        self.recorder.append(spaceop)
 
     def do_op(self, op):
+        self.maybe_merge()
         self.record(op)
         self.guessexception(op.canraise)
         return op.result
 
-    def guessexception(self, exceptions, force=False):
+    def guessexception(self, exceptions):
         """
         Catch possible exceptions implicitly.
         """
         if not exceptions:
             return
-        if not force and not any(isinstance(block, (ExceptBlock, FinallyBlock))
-                                 for block in self.blockstack):
-            # The implicit exception wouldn't be caught and would later get
-            # removed, so don't bother creating it.
-            return
-        self.recorder.guessexception(self, *exceptions)
+        # Implicit exceptions are ignored unless they are caught explicitly
+        if self.has_exc_handler():
+            self.recorder.guessexception(self, *exceptions)
+
+    def has_exc_handler(self):
+        return any(isinstance(block, (ExceptBlock, FinallyBlock))
+                for block in self.blockstack)
 
     def build_flow(self):
         graph = self.graph
@@ -430,35 +409,8 @@
             while True:
                 next_offset = self.handle_bytecode(next_offset)
                 self.recorder.final_state = self.getstate(next_offset)
-
-        except RaiseImplicit as e:
-            w_exc = e.w_exc
-            if isinstance(w_exc.w_type, Constant):
-                exc_cls = w_exc.w_type.value
-            else:
-                exc_cls = Exception
-            msg = "implicit %s shouldn't occur" % exc_cls.__name__
-            w_type = Constant(AssertionError)
-            w_value = Constant(AssertionError(msg))
-            link = Link([w_type, w_value], self.graph.exceptblock)
-            self.recorder.crnt_block.closeblock(link)
-
-        except Raise as e:
-            w_exc = e.w_exc
-            if w_exc.w_type == const(ImportError):
-                msg = 'import statement always raises %s' % e
-                raise ImportError(msg)
-            link = Link([w_exc.w_type, w_exc.w_value], self.graph.exceptblock)
-            self.recorder.crnt_block.closeblock(link)
-
         except StopFlowing:
             pass
-
-        except Return as exc:
-            w_result = exc.w_value
-            link = Link([w_result], self.graph.returnblock)
-            self.recorder.crnt_block.closeblock(link)
-
         except FlowingError as exc:
             if exc.ctx is None:
                 exc.ctx = self
@@ -476,14 +428,8 @@
             if newstate is not None:
                 break
         else:
-            newstate = currentstate.copy()
-            newblock = SpamBlock(newstate)
-            # unconditionally link the current block to the newblock
-            outputargs = currentstate.getoutputargs(newstate)
-            link = Link(outputargs, newblock)
-            currentblock.closeblock(link)
+            newblock = self.make_next_block(currentblock, currentstate)
             candidates.insert(0, newblock)
-            self.pendingblocks.append(newblock)
             return
 
         if newstate.matches(block.framestate):
@@ -493,7 +439,7 @@
 
         newblock = SpamBlock(newstate)
         varnames = self.pycode.co_varnames
-        for name, w_value in zip(varnames, newstate.mergeable):
+        for name, w_value in zip(varnames, newstate.locals_w):
             if isinstance(w_value, Variable):
                 w_value.rename(name)
         # unconditionally link the current block to the newblock
@@ -513,11 +459,21 @@
         candidates.insert(0, newblock)
         self.pendingblocks.append(newblock)
 
+    def make_next_block(self, block, state):
+        newstate = state.copy()
+        newblock = SpamBlock(newstate)
+        # unconditionally link the current block to the newblock
+        outputargs = state.getoutputargs(newstate)
+        link = Link(outputargs, newblock)
+        block.closeblock(link)
+        self.pendingblocks.append(newblock)
+        return newblock
+
     # hack for unrolling iterables, don't use this
     def replace_in_stack(self, oldvalue, newvalue):
         w_new = Constant(newvalue)
-        stack_items_w = self.locals_stack_w
-        for i in range(self.valuestackdepth - 1, self.pycode.co_nlocals - 1, -1):
+        stack_items_w = self.stack
+        for i in range(self.stackdepth - 1, - 1, -1):
             w_v = stack_items_w[i]
             if isinstance(w_v, Constant):
                 if w_v.value is oldvalue:
@@ -541,7 +497,7 @@
             if isinstance(signal, block.handles):
                 return block.handle(self, signal)
             block.cleanupstack(self)
-        return signal.nomoreblocks()
+        return signal.nomoreblocks(self)
 
     def getlocalvarname(self, index):
         return self.pycode.co_varnames[index]
@@ -870,7 +826,7 @@
             op.simple_call(w_exitfunc, w_None, w_None, w_None).eval(self)
 
     def LOAD_FAST(self, varindex):
-        w_value = self.locals_stack_w[varindex]
+        w_value = self.locals_w[varindex]
         if w_value is None:
             raise FlowingError("Local variable referenced before assignment")
         self.pushvalue(w_value)
@@ -915,7 +871,7 @@
     def STORE_FAST(self, varindex):
         w_newvalue = self.popvalue()
         assert w_newvalue is not None
-        self.locals_stack_w[varindex] = w_newvalue
+        self.locals_w[varindex] = w_newvalue
         if isinstance(w_newvalue, Variable):
             w_newvalue.rename(self.getlocalvarname(varindex))
 
@@ -1128,11 +1084,11 @@
         op.simple_call(w_append_meth, w_value).eval(self)
 
     def DELETE_FAST(self, varindex):
-        if self.locals_stack_w[varindex] is None:
+        if self.locals_w[varindex] is None:
             varname = self.getlocalvarname(varindex)
             message = "local variable '%s' referenced before assignment"
             raise UnboundLocalError(message, varname)
-        self.locals_stack_w[varindex] = None
+        self.locals_w[varindex] = None
 
     def STORE_MAP(self, oparg):
         w_key = self.popvalue()
@@ -1220,25 +1176,32 @@
                 WHY_CONTINUE,   Continue
                 WHY_YIELD       not needed
     """
-    def nomoreblocks(self):
+    def nomoreblocks(self, ctx):
         raise BytecodeCorruption("misplaced bytecode - should not return")
 
+    def __eq__(self, other):
+        return type(other) is type(self) and other.args == self.args
+
 
 class Return(FlowSignal):
     """Signals a 'return' statement.
-    Argument is the wrapped object to return."""
-
+    Argument is the wrapped object to return.
+    """
     def __init__(self, w_value):
         self.w_value = w_value
 
-    def nomoreblocks(self):
-        raise Return(self.w_value)
+    def nomoreblocks(self, ctx):
+        w_result = self.w_value
+        link = Link([w_result], ctx.graph.returnblock)
+        ctx.recorder.crnt_block.closeblock(link)
+        raise StopFlowing
 
-    def state_unpack_variables(self):
+    @property
+    def args(self):
         return [self.w_value]
 
     @staticmethod
-    def state_pack_variables(w_value):
+    def rebuild(w_value):
         return Return(w_value)
 
 class Raise(FlowSignal):
@@ -1248,28 +1211,48 @@
     def __init__(self, w_exc):
         self.w_exc = w_exc
 
-    def nomoreblocks(self):
-        raise self
+    def nomoreblocks(self, ctx):
+        w_exc = self.w_exc
+        if w_exc.w_type == const(ImportError):
+            msg = 'import statement always raises %s' % self
+            raise ImportError(msg)
+        link = Link([w_exc.w_type, w_exc.w_value], ctx.graph.exceptblock)
+        ctx.recorder.crnt_block.closeblock(link)
+        raise StopFlowing
 
-    def state_unpack_variables(self):
+    @property
+    def args(self):
         return [self.w_exc.w_type, self.w_exc.w_value]
 
-    @staticmethod
-    def state_pack_variables(w_type, w_value):
-        return Raise(FSException(w_type, w_value))
+    @classmethod
+    def rebuild(cls, w_type, w_value):
+        return cls(FSException(w_type, w_value))
 
 class RaiseImplicit(Raise):
     """Signals an exception raised implicitly"""
+    def nomoreblocks(self, ctx):
+        w_exc = self.w_exc
+        if isinstance(w_exc.w_type, Constant):
+            exc_cls = w_exc.w_type.value
+        else:
+            exc_cls = Exception
+        msg = "implicit %s shouldn't occur" % exc_cls.__name__
+        w_type = Constant(AssertionError)
+        w_value = Constant(AssertionError(msg))
+        link = Link([w_type, w_value], ctx.graph.exceptblock)
+        ctx.recorder.crnt_block.closeblock(link)
+        raise StopFlowing
 
 
 class Break(FlowSignal):
     """Signals a 'break' statement."""
 
-    def state_unpack_variables(self):
+    @property
+    def args(self):
         return []
 
     @staticmethod
-    def state_pack_variables():
+    def rebuild():
         return Break.singleton
 
 Break.singleton = Break()
@@ -1281,11 +1264,12 @@
     def __init__(self, jump_to):
         self.jump_to = jump_to
 
-    def state_unpack_variables(self):
+    @property
+    def args(self):
         return [const(self.jump_to)]
 
     @staticmethod
-    def state_pack_variables(w_jump_to):
+    def rebuild(w_jump_to):
         return Continue(w_jump_to.value)
 
 
@@ -1295,21 +1279,21 @@
 
     def __init__(self, ctx, handlerposition):
         self.handlerposition = handlerposition
-        self.valuestackdepth = ctx.valuestackdepth
+        self.stackdepth = ctx.stackdepth
 
     def __eq__(self, other):
         return (self.__class__ is other.__class__ and
                 self.handlerposition == other.handlerposition and
-                self.valuestackdepth == other.valuestackdepth)
+                self.stackdepth == other.stackdepth)
 
     def __ne__(self, other):
         return not (self == other)
 
     def __hash__(self):
-        return hash((self.handlerposition, self.valuestackdepth))
+        return hash((self.handlerposition, self.stackdepth))
 
     def cleanupstack(self, ctx):
-        ctx.dropvaluesuntil(self.valuestackdepth)
+        ctx.dropvaluesuntil(self.stackdepth)
 
     def handle(self, ctx, unroller):
         raise NotImplementedError
diff --git a/rpython/flowspace/framestate.py b/rpython/flowspace/framestate.py
--- a/rpython/flowspace/framestate.py
+++ b/rpython/flowspace/framestate.py
@@ -1,21 +1,50 @@
-from rpython.flowspace.model import Variable, Constant
+from rpython.flowspace.model import Variable, Constant, FSException
 from rpython.rlib.unroll import SpecTag
 
+def _copy(v):
+    from rpython.flowspace.flowcontext import FlowSignal
+    if isinstance(v, Variable):
+        return Variable(v)
+    elif isinstance(v, FlowSignal):
+        vars = [_copy(var) for var in v.args]
+        return v.rebuild(*vars)
+    else:
+        return v
+
+def _union(seq1, seq2):
+    return [union(v1, v2) for v1, v2 in zip(seq1, seq2)]
+
 
 class FrameState(object):
-    def __init__(self, mergeable, blocklist, next_offset):
-        self.mergeable = mergeable
+    def __init__(self, locals_w, stack, last_exception, blocklist, next_offset):
+        self.locals_w = locals_w
+        self.stack = stack
+        self.last_exception = last_exception
         self.blocklist = blocklist
         self.next_offset = next_offset
+        self._mergeable = None
+
+    @property
+    def mergeable(self):
+        if self._mergeable is not None:
+            return self._mergeable
+        self._mergeable = data = self.locals_w + self.stack
+        if self.last_exception is None:
+            data.append(Constant(None))
+            data.append(Constant(None))
+        else:
+            data.append(self.last_exception.w_type)
+            data.append(self.last_exception.w_value)
+        recursively_flatten(data)
+        return data
 
     def copy(self):
         "Make a copy of this state in which all Variables are fresh."
-        newstate = []
-        for w in self.mergeable:
-            if isinstance(w, Variable):
-                w = Variable(w)
-            newstate.append(w)
-        return FrameState(newstate, self.blocklist, self.next_offset)
+        exc = self.last_exception
+        if exc is not None:
+            exc = FSException(_copy(exc.w_type), _copy(exc.w_value))
+        return FrameState(map(_copy, self.locals_w), map(_copy, self.stack),
+                exc, self.blocklist, self.next_offset)
 
     def getvariables(self):
         return [w for w in self.mergeable if isinstance(w, Variable)]
@@ -33,18 +62,31 @@
                 return False
         return True
 
+    def _exc_args(self):
+        if self.last_exception is None:
+            return [Constant(None), Constant(None)]
+        else:
+            return [self.last_exception.w_type,
+                    self.last_exception.w_value]
+
     def union(self, other):
         """Compute a state that is at least as general as both self and other.
            A state 'a' is more general than a state 'b' if all Variables in 'b'
            are also Variables in 'a', but 'a' may have more Variables.
         """
-        newstate = []
         try:
-            for w1, w2 in zip(self.mergeable, other.mergeable):
-                newstate.append(union(w1, w2))
+            locals = _union(self.locals_w, other.locals_w)
+            stack = _union(self.stack, other.stack)
+            if self.last_exception is None and other.last_exception is None:
+                exc = None
+            else:
+                args1 = self._exc_args()
+                args2 = other._exc_args()
+                exc = FSException(union(args1[0], args2[0]),
+                        union(args1[1], args2[1]))
         except UnionError:
             return None
-        return FrameState(newstate, self.blocklist, self.next_offset)
+        return FrameState(locals, stack, exc, self.blocklist, self.next_offset)
 
     def getoutputargs(self, targetstate):
         "Return the output arguments needed to link self to targetstate."
@@ -61,6 +103,7 @@
 
 def union(w1, w2):
     "Union of two variables or constants."
+    from rpython.flowspace.flowcontext import FlowSignal
     if w1 == w2:
         return w1
     if w1 is None or w2 is None:
@@ -69,38 +112,21 @@
     if isinstance(w1, Variable) or isinstance(w2, Variable):
         return Variable()  # new fresh Variable
     if isinstance(w1, Constant) and isinstance(w2, Constant):
-        # FlowSignal represent stack unrollers in the stack.
-        # They should not be merged because they will be unwrapped.
-        # This is needed for try:except: and try:finally:, though
-        # it makes the control flow a bit larger by duplicating the
-        # handlers.
-        dont_merge_w1 = w1 in UNPICKLE_TAGS or isinstance(w1.value, SpecTag)
-        dont_merge_w2 = w2 in UNPICKLE_TAGS or isinstance(w2.value, SpecTag)
-        if dont_merge_w1 or dont_merge_w2:
+        if isinstance(w1.value, SpecTag) or isinstance(w2.value, SpecTag):
             raise UnionError
         else:
             return Variable()  # generalize different constants
+    if isinstance(w1, FlowSignal) and isinstance(w2, FlowSignal):
+        if type(w1) is not type(w2):
+            raise UnionError
+        vars = [union(v1, v2) for v1, v2 in zip(w1.args, w2.args)]
+        return w1.rebuild(*vars)
+    if isinstance(w1, FlowSignal) or isinstance(w2, FlowSignal):
+        raise UnionError
     raise TypeError('union of %r and %r' % (w1.__class__.__name__,
                                             w2.__class__.__name__))
 
 
-# ____________________________________________________________
-#
-# We have to flatten out the state of the frame into a list of
-# Variables and Constants.  This is done above by collecting the
-# locals and the items on the value stack, but the latter may contain
-# FlowSignal.  We have to handle these specially, because
-# some of them hide references to more Variables and Constants.
-# The trick is to flatten ("pickle") them into the list so that the
-# extra Variables show up directly in the list too.
-
-class PickleTag:
-    pass
-
-PICKLE_TAGS = {}
-UNPICKLE_TAGS = {}
-
-
 def recursively_flatten(lst):
     from rpython.flowspace.flowcontext import FlowSignal
     i = 0
@@ -109,22 +135,4 @@
         if not isinstance(unroller, FlowSignal):
             i += 1
         else:
-            vars = unroller.state_unpack_variables()
-            key = unroller.__class__, len(vars)
-            try:
-                tag = PICKLE_TAGS[key]
-            except KeyError:
-                tag = PICKLE_TAGS[key] = Constant(PickleTag())
-                UNPICKLE_TAGS[tag] = key
-            lst[i:i + 1] = [tag] + vars
-
-
-def recursively_unflatten(lst):
-    for i in xrange(len(lst) - 1, -1, -1):
-        item = lst[i]
-        if item in UNPICKLE_TAGS:
-            unrollerclass, argcount = UNPICKLE_TAGS[item]
-            arguments = lst[i + 1:i + 1 + argcount]
-            del lst[i + 1:i + 1 + argcount]
-            unroller = unrollerclass.state_pack_variables(*arguments)
-            lst[i] = unroller
+            lst[i:i + 1] = unroller.args
diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py
--- a/rpython/flowspace/operation.py
+++ b/rpython/flowspace/operation.py
@@ -517,7 +517,7 @@
                     ctx.replace_in_stack(it, next_unroller)
                     return const(v)
         w_item = ctx.do_op(self)
-        ctx.guessexception([StopIteration, RuntimeError], force=True)
+        ctx.recorder.guessexception(ctx, StopIteration, RuntimeError)
         return w_item
 
 class GetAttr(SingleDispatchMixin, HLOperation):
diff --git a/rpython/flowspace/pygraph.py b/rpython/flowspace/pygraph.py
--- a/rpython/flowspace/pygraph.py
+++ b/rpython/flowspace/pygraph.py
@@ -11,10 +11,10 @@
 
     def __init__(self, func, code):
         from rpython.flowspace.flowcontext import SpamBlock
-        data = [None] * code.co_nlocals
+        locals = [None] * code.co_nlocals
         for i in range(code.formalargcount):
-            data[i] = Variable(code.co_varnames[i])
-        state = FrameState(data + [Constant(None), Constant(None)], [], 0)
+            locals[i] = Variable(code.co_varnames[i])
+        state = FrameState(locals, [], None, [], 0)
         initialblock = SpamBlock(state)
         super(PyGraph, self).__init__(self._sanitize_funcname(func), initialblock)
         self.func = func
diff --git a/rpython/flowspace/test/test_flowcontext.py b/rpython/flowspace/test/test_flowcontext.py
new file mode 100644
--- /dev/null
+++ b/rpython/flowspace/test/test_flowcontext.py
@@ -0,0 +1,15 @@
+""" Unit tests for flowcontext.py """
+import pytest
+from rpython.flowspace.model import Variable, FSException
+from rpython.flowspace.flowcontext import (
+    Return, Raise, RaiseImplicit, Continue, Break)
+
+ at pytest.mark.parametrize('signal', [
+    Return(Variable()),
+    Raise(FSException(Variable(), Variable())),
+    RaiseImplicit(FSException(Variable(), Variable())),
+    Break(),
+    Continue(42),
+])
+def test_signals(signal):
+    assert signal.rebuild(*signal.args) == signal
diff --git a/rpython/flowspace/test/test_framestate.py b/rpython/flowspace/test/test_framestate.py
--- a/rpython/flowspace/test/test_framestate.py
+++ b/rpython/flowspace/test/test_framestate.py
@@ -15,7 +15,7 @@
         ctx = FlowContext(graph, code)
         # hack the frame
         ctx.setstate(graph.startblock.framestate)
-        ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Constant(None)
+        ctx.locals_w[-1] = Constant(None)
         return ctx
 
     def func_simple(x):
@@ -31,7 +31,7 @@
     def test_neq_hacked_framestate(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
-        ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable()
+        ctx.locals_w[-1] = Variable()
         fs2 = ctx.getstate(0)
         assert not fs1.matches(fs2)
 
@@ -44,7 +44,7 @@
     def test_union_on_hacked_framestates(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
-        ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable()
+        ctx.locals_w[-1] = Variable()
         fs2 = ctx.getstate(0)
         assert fs1.union(fs2).matches(fs2)  # fs2 is more general
         assert fs2.union(fs1).matches(fs2)  # fs2 is more general
@@ -52,7 +52,7 @@
     def test_restore_frame(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
-        ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable()
+        ctx.locals_w[-1] = Variable()
         ctx.setstate(fs1)
         assert fs1.matches(ctx.getstate(0))
 
@@ -71,26 +71,25 @@
     def test_getoutputargs(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
-        ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable()
+        ctx.locals_w[-1] = Variable()
         fs2 = ctx.getstate(0)
         outputargs = fs1.getoutputargs(fs2)
         # 'x' -> 'x' is a Variable
         # locals_w[n-1] -> locals_w[n-1] is Constant(None)
-        assert outputargs == [ctx.locals_stack_w[0], Constant(None)]
+        assert outputargs == [ctx.locals_w[0], Constant(None)]
 
     def test_union_different_constants(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
-        ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Constant(42)
+        ctx.locals_w[-1] = Constant(42)
         fs2 = ctx.getstate(0)
         fs3 = fs1.union(fs2)
         ctx.setstate(fs3)
-        assert isinstance(ctx.locals_stack_w[ctx.pycode.co_nlocals-1],
-                          Variable)   # generalized
+        assert isinstance(ctx.locals_w[-1], Variable)   # generalized
 
     def test_union_spectag(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
-        ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Constant(SpecTag())
+        ctx.locals_w[-1] = Constant(SpecTag())
         fs2 = ctx.getstate(0)
         assert fs1.union(fs2) is None   # UnionError
diff --git a/rpython/jit/backend/arm/callbuilder.py b/rpython/jit/backend/arm/callbuilder.py
--- a/rpython/jit/backend/arm/callbuilder.py
+++ b/rpython/jit/backend/arm/callbuilder.py
@@ -176,11 +176,14 @@
 
     def write_real_errno(self, save_err):
         if save_err & rffi.RFFI_READSAVED_ERRNO:
-            # Just before a call, read 'rpy_errno' and write it into the
+            # Just before a call, read '*_errno' and write it into the
             # real 'errno'.  The r0-r3 registers contain arguments to the
             # future call; the r5-r7 registers contain various stuff.
             # We still have r8-r12.
-            rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu)
+            if save_err & rffi.RFFI_ALT_ERRNO:
+                rpy_errno = llerrno.get_alt_errno_offset(self.asm.cpu)
+            else:
+                rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu)
             p_errno = llerrno.get_p_errno_offset(self.asm.cpu)
             self.mc.LDR_ri(r.r9.value, r.sp.value,
                            self.asm.saved_threadlocal_addr + self.current_sp)
@@ -199,10 +202,13 @@
     def read_real_errno(self, save_err):
         if save_err & rffi.RFFI_SAVE_ERRNO:
             # Just after a call, read the real 'errno' and save a copy of
-            # it inside our thread-local 'rpy_errno'.  Registers r8-r12
+            # it inside our thread-local '*_errno'.  Registers r8-r12
             # are unused here, and registers r2-r3 never contain anything
             # after the call.
-            rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu)
+            if save_err & rffi.RFFI_ALT_ERRNO:
+                rpy_errno = llerrno.get_alt_errno_offset(self.asm.cpu)
+            else:
+                rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu)
             p_errno = llerrno.get_p_errno_offset(self.asm.cpu)
             self.mc.LDR_ri(r.r3.value, r.sp.value,
                            self.asm.saved_threadlocal_addr)
diff --git a/rpython/jit/backend/llsupport/llerrno.py b/rpython/jit/backend/llsupport/llerrno.py
--- a/rpython/jit/backend/llsupport/llerrno.py
+++ b/rpython/jit/backend/llsupport/llerrno.py
@@ -18,19 +18,41 @@
         return 3 * WORD
 
 
+def get_debug_saved_alterrno(cpu):
+    return cpu._debug_errno_container[4]
+
+def set_debug_saved_alterrno(cpu, nerrno):
+    assert nerrno >= 0
+    cpu._debug_errno_container[4] = nerrno
+
+def get_alt_errno_offset(cpu):
+    if cpu.translate_support_code:
+        from rpython.rlib import rthread
+        return rthread.tlfield_alt_errno.getoffset()
+    else:
+        return 4 * WORD
+
+
 def get_debug_saved_lasterror(cpu):
-    return cpu._debug_errno_container[4]
+    return cpu._debug_errno_container[5]
 
 def set_debug_saved_lasterror(cpu, nerrno):
     assert nerrno >= 0
-    cpu._debug_errno_container[4] = nerrno
+    cpu._debug_errno_container[5] = nerrno
 
 def get_rpy_lasterror_offset(cpu):
     if cpu.translate_support_code:
         from rpython.rlib import rthread
         return rthread.tlfield_rpy_lasterror.getoffset()
     else:
-        return 4 * WORD
+        return 5 * WORD
+
+def get_alt_lasterror_offset(cpu):
+    if cpu.translate_support_code:
+        from rpython.rlib import rthread
+        return rthread.tlfield_alt_lasterror.getoffset()
+    else:
+        return 6 * WORD
 
 
 def _fetch_addr_errno():
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -63,7 +63,7 @@
                                               ad.lendescr, FLAG_FLOAT)
         self.setup()
         self._debug_errno_container = lltype.malloc(
-            rffi.CArray(lltype.Signed), 5, flavor='raw', zero=True,
+            rffi.CArray(lltype.Signed), 7, flavor='raw', zero=True,
             track_allocation=False)
 
     def getarraydescr_for_frame(self, type):
diff --git a/rpython/jit/backend/llsupport/test/zrpy_releasegil_test.py b/rpython/jit/backend/llsupport/test/zrpy_releasegil_test.py
--- a/rpython/jit/backend/llsupport/test/zrpy_releasegil_test.py
+++ b/rpython/jit/backend/llsupport/test/zrpy_releasegil_test.py
@@ -98,6 +98,7 @@
         self.run('close_stack')
         assert 'call_release_gil' in udir.join('TestCompileFramework.log').read()
 
+    # XXX this should also test get/set_alterrno ?
     def define_get_set_errno(self):
         eci = ExternalCompilationInfo(
             post_include_bits=[r'''
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -2948,7 +2948,11 @@
         calldescr = self.cpu._calldescr_dynamic_for_tests([types.slong]*7,
                                                           types.slong)
         #
-        for saveerr in [rffi.RFFI_ERR_NONE, rffi.RFFI_SAVE_ERRNO]:
+        for saveerr in [rffi.RFFI_ERR_NONE,
+                        rffi.RFFI_SAVE_ERRNO,
+                        rffi.RFFI_ERR_NONE | rffi.RFFI_ALT_ERRNO,
+                        rffi.RFFI_SAVE_ERRNO | rffi.RFFI_ALT_ERRNO,
+                        ]:
             faildescr = BasicFailDescr(1)
             inputargs = [BoxInt() for i in range(7)]
             i1 = BoxInt()
@@ -2965,15 +2969,23 @@
             self.cpu.compile_loop(inputargs, ops, looptoken)
             #
             llerrno.set_debug_saved_errno(self.cpu, 24)
+            llerrno.set_debug_saved_alterrno(self.cpu, 25)
             deadframe = self.cpu.execute_token(looptoken, 9, 8, 7, 6, 5, 4, 3)
             original_result = self.cpu.get_int_value(deadframe, 0)
             result = llerrno.get_debug_saved_errno(self.cpu)
-            print 'saveerr =', saveerr, ': got result =', result
+            altresult = llerrno.get_debug_saved_alterrno(self.cpu)
+            print 'saveerr =', saveerr, ': got result =', result, \
+                  'altresult =', altresult
             #
-            if saveerr == rffi.RFFI_SAVE_ERRNO:
-                assert result == 42      # from the C code
-            else:
-                assert result == 24      # not touched
+            expected = {
+                rffi.RFFI_ERR_NONE: (24, 25),
+                rffi.RFFI_SAVE_ERRNO: (42, 25),
+                rffi.RFFI_ERR_NONE | rffi.RFFI_ALT_ERRNO: (24, 25),
+                rffi.RFFI_SAVE_ERRNO | rffi.RFFI_ALT_ERRNO: (24, 42),
+            }
+            # expected (24, 25) as originally set, with possibly one
+            # of the two changed to 42 by the assembler code
+            assert (result, altresult) == expected[saveerr]
             assert original_result == 3456789
 
     def test_call_release_gil_readsaved_errno(self):
@@ -3007,7 +3019,11 @@
         calldescr = self.cpu._calldescr_dynamic_for_tests([types.slong]*7,
                                                           types.slong)
         #
-        for saveerr in [rffi.RFFI_READSAVED_ERRNO, rffi.RFFI_ZERO_ERRNO_BEFORE]:
+        for saveerr in [rffi.RFFI_READSAVED_ERRNO,
+                        rffi.RFFI_ZERO_ERRNO_BEFORE,
+                        rffi.RFFI_READSAVED_ERRNO   | rffi.RFFI_ALT_ERRNO,
+                        rffi.RFFI_ZERO_ERRNO_BEFORE | rffi.RFFI_ALT_ERRNO,
+                        ]:
             faildescr = BasicFailDescr(1)
             inputargs = [BoxInt() for i in range(7)]
             i1 = BoxInt()
@@ -3024,12 +3040,17 @@
             self.cpu.compile_loop(inputargs, ops, looptoken)
             #
             llerrno.set_debug_saved_errno(self.cpu, 24)
+            llerrno.set_debug_saved_alterrno(self.cpu, 25)
             deadframe = self.cpu.execute_token(looptoken, 9, 8, 7, 6, 5, 4, 3)
             result = self.cpu.get_int_value(deadframe, 0)
             assert llerrno.get_debug_saved_errno(self.cpu) == 24
+            assert llerrno.get_debug_saved_alterrno(self.cpu) == 25
             #
-            if saveerr == rffi.RFFI_READSAVED_ERRNO:
-                assert result == 24 + 345678900
+            if saveerr & rffi.RFFI_READSAVED_ERRNO:
+                if saveerr & rffi.RFFI_ALT_ERRNO:
+                    assert result == 25 + 345678900
+                else:
+                    assert result == 24 + 345678900
             else:
                 assert result == 0  + 345678900
 
@@ -3064,7 +3085,10 @@
                                                           types.slong)
         #
         for saveerr in [rffi.RFFI_SAVE_ERRNO,  # but not _LASTERROR
-                        rffi.RFFI_SAVE_LASTERROR]:
+                        rffi.RFFI_SAVE_ERRNO | rffi.RFFI_ALT_ERRNO,
+                        rffi.RFFI_SAVE_LASTERROR,
+                        rffi.RFFI_SAVE_LASTERROR | rffi.RFFI_ALT_ERRNO,
+                        ]:
             faildescr = BasicFailDescr(1)
             inputargs = [BoxInt() for i in range(7)]
             i1 = BoxInt()
@@ -3125,7 +3149,9 @@
         calldescr = self.cpu._calldescr_dynamic_for_tests([types.slong]*7,
                                                           types.slong)
         #
-        for saveerr in [rffi.RFFI_READSAVED_LASTERROR]:
+        for saveerr in [rffi.RFFI_READSAVED_LASTERROR,
+                        rffi.RFFI_READSAVED_LASTERROR | rffi.RFFI_ALT_ERRNO,
+                       ]:
             faildescr = BasicFailDescr(1)
             inputargs = [BoxInt() for i in range(7)]
             i1 = BoxInt()
@@ -3198,7 +3224,10 @@
         calldescr = self.cpu._calldescr_dynamic_for_tests([types.slong]*7,
                                                           types.slong)
         #
-        for saveerr in [rffi.RFFI_ERR_ALL]:
+        for saveerr in [rffi.RFFI_ERR_ALL,
+                        rffi.RFFI_ERR_ALL | rffi.RFFI_ALT_ERRNO, 
+                       ]:
+            use_alt_errno = saveerr & rffi.RFFI_ALT_ERRNO
             faildescr = BasicFailDescr(1)
             inputargs = [BoxInt() for i in range(7)]
             i1 = BoxInt()
@@ -3214,7 +3243,10 @@
             looptoken = JitCellToken()
             self.cpu.compile_loop(inputargs, ops, looptoken)
             #
-            llerrno.set_debug_saved_errno(self.cpu, 8)
+            if use_alt_errno:
+                llerrno.set_debug_saved_alterrno(self.cpu, 8)
+            else:
+                llerrno.set_debug_saved_errno(self.cpu, 8)
             llerrno.set_debug_saved_lasterror(self.cpu, 9)
             deadframe = self.cpu.execute_token(looptoken, 1, 2, 3, 4, 5, 6, 7)
             result = self.cpu.get_int_value(deadframe, 0)
diff --git a/rpython/jit/backend/x86/callbuilder.py b/rpython/jit/backend/x86/callbuilder.py
--- a/rpython/jit/backend/x86/callbuilder.py
+++ b/rpython/jit/backend/x86/callbuilder.py
@@ -196,21 +196,27 @@
             SetLastError_addr = self.asm.cpu.cast_adr_to_int(adr)
             assert isinstance(self, CallBuilder32)    # Windows 32-bit only
             #
-            rpy_lasterror = llerrno.get_rpy_lasterror_offset(self.asm.cpu)
+            if save_err & rffi.RFFI_ALT_ERRNO:
+                lasterror = llerrno.get_alt_lasterror_offset(self.asm.cpu)
+            else:
+                lasterror = llerrno.get_rpy_lasterror_offset(self.asm.cpu)
             tlofsreg = self.get_tlofs_reg()    # => esi, callee-saved
             self.save_stack_position()         # => edi, callee-saved
-            mc.PUSH_m((tlofsreg.value, rpy_lasterror))
+            mc.PUSH_m((tlofsreg.value, lasterror))
             mc.CALL(imm(SetLastError_addr))
             # restore the stack position without assuming a particular
             # calling convention of _SetLastError()
             self.mc.MOV(esp, self.saved_stack_position_reg)
 
         if save_err & rffi.RFFI_READSAVED_ERRNO:
-            # Just before a call, read 'rpy_errno' and write it into the
+            # Just before a call, read '*_errno' and write it into the
             # real 'errno'.  Most registers are free here, including the
             # callee-saved ones, except 'ebx' and except the ones used to
             # pass the arguments on x86-64.
-            rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu)
+            if save_err & rffi.RFFI_ALT_ERRNO:
+                rpy_errno = llerrno.get_alt_errno_offset(self.asm.cpu)
+            else:
+                rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu)
             p_errno = llerrno.get_p_errno_offset(self.asm.cpu)
             tlofsreg = self.get_tlofs_reg()    # => esi or r12, callee-saved
             if IS_X86_32:
@@ -234,11 +240,14 @@
 
         if save_err & rffi.RFFI_SAVE_ERRNO:
             # Just after a call, read the real 'errno' and save a copy of
-            # it inside our thread-local 'rpy_errno'.  Most registers are
+            # it inside our thread-local '*_errno'.  Most registers are
             # free here, including the callee-saved ones, except 'ebx'.
             # The tlofs register might have been loaded earlier and is
             # callee-saved, so it does not need to be reloaded.
-            rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu)
+            if save_err & rffi.RFFI_ALT_ERRNO:
+                rpy_errno = llerrno.get_alt_errno_offset(self.asm.cpu)
+            else:
+                rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu)
             p_errno = llerrno.get_p_errno_offset(self.asm.cpu)
             tlofsreg = self.get_tlofs_reg()   # => esi or r12 (possibly reused)
             mc.MOV_rm(edi.value, (tlofsreg.value, p_errno))
@@ -256,13 +265,16 @@
             GetLastError_addr = self.asm.cpu.cast_adr_to_int(adr)
             assert isinstance(self, CallBuilder32)    # Windows 32-bit only
             #
-            rpy_lasterror = llerrno.get_rpy_lasterror_offset(self.asm.cpu)
+            if save_err & rffi.RFFI_ALT_ERRNO:
+                lasterror = llerrno.get_alt_lasterror_offset(self.asm.cpu)
+            else:
+                lasterror = llerrno.get_rpy_lasterror_offset(self.asm.cpu)
             self.save_result_value(save_edx=True)   # save eax/edx/xmm0
             self.result_value_saved_early = True
             mc.CALL(imm(GetLastError_addr))
             #
             tlofsreg = self.get_tlofs_reg()    # => esi (possibly reused)
-            mc.MOV32_mr((tlofsreg.value, rpy_lasterror), eax.value)
+            mc.MOV32_mr((tlofsreg.value, lasterror), eax.value)
 
     def move_real_result_and_call_reacqgil_addr(self, fastgil):
         from rpython.jit.backend.x86 import rx86
@@ -314,8 +326,8 @@
             # in 'ebx'), and if not, we fall back to 'reacqgil_addr'.
             mc.J_il8(rx86.Conditions['NE'], 0)
             jne_location = mc.get_relative_pos()
-            # here, ecx is zero (so rpy_fastgil was in 'released' state
-            # before the XCHG, but the XCHG acquired it by writing 1)
+            # here, ecx (=old_value) is zero (so rpy_fastgil was in 'released'
+            # state before the XCHG, but the XCHG acquired it by writing 1)
             rst = gcrootmap.get_root_stack_top_addr()
             mc = self.mc
             mc.CMP(ebx, heap(rst))
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -749,18 +749,14 @@
             rstack._stack_criticalcode_stop()
 
     def handle_async_forcing(self, deadframe):
-        from rpython.jit.metainterp.resume import (force_from_resumedata,
-                                                   AlreadyForced)
+        from rpython.jit.metainterp.resume import force_from_resumedata
         metainterp_sd = self.metainterp_sd
         vinfo = self.jitdriver_sd.virtualizable_info
         ginfo = self.jitdriver_sd.greenfield_info
         # there is some chance that this is already forced. In this case
         # the virtualizable would have a token = NULL
-        try:
-            all_virtuals = force_from_resumedata(metainterp_sd, self, deadframe,
-                                                 vinfo, ginfo)
-        except AlreadyForced:
-            return
+        all_virtuals = force_from_resumedata(metainterp_sd, self, deadframe,
+                                             vinfo, ginfo)
         # The virtualizable data was stored on the real virtualizable above.
         # Handle all_virtuals: keep them for later blackholing from the
         # future failure of the GUARD_NOT_FORCED
diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -1,61 +1,111 @@
 from rpython.jit.metainterp.history import ConstInt
 from rpython.jit.metainterp.resoperation import rop
 
+class HeapCacheValue(object):
+    def __init__(self, box):
+        self.box = box
+        self.likely_virtual = False
+        self.reset_keep_likely_virtual()
+
+    def reset_keep_likely_virtual(self):
+        self.known_class = False
+        # did we see the allocation during tracing?
+        self.seen_allocation = False
+        self.is_unescaped = False
+        self.nonstandard_virtualizable = False
+        self.length = None
+        self.dependencies = None
+
+    def __repr__(self):
+        return 'HeapCacheValue(%s)' % (self.box, )
+
+
+class CacheEntry(object):
+    def __init__(self):
+        # both are {from_value: to_value} dicts
+        # the first is for boxes where we did not see the allocation, the
+        # second for anything else. the reason that distinction makes sense is
+        # because if we saw the allocation, we know it cannot alias with
+        # anything else where we saw the allocation.
+        self.cache_anything = {}
+        self.cache_seen_allocation = {}
+
+    def _clear_cache_on_write(self, seen_allocation_of_target):
+        if not seen_allocation_of_target:
+            self.cache_seen_allocation.clear()
+        self.cache_anything.clear()
+
+    def _getdict(self, value):
+        if value.seen_allocation:
+            return self.cache_seen_allocation
+        else:
+            return self.cache_anything
+
+    def do_write_with_aliasing(self, value, fieldvalue):
+        self._clear_cache_on_write(value.seen_allocation)
+        self._getdict(value)[value] = fieldvalue
+
+    def read(self, value):
+        return self._getdict(value).get(value, None)
+
+    def read_now_known(self, value, fieldvalue):
+        self._getdict(value)[value] = fieldvalue
+
+    def invalidate_unescaped(self):
+        self._invalidate_unescaped(self.cache_anything)
+        self._invalidate_unescaped(self.cache_seen_allocation)
+
+    def _invalidate_unescaped(self, d):
+        for value in d.keys():
+            if not value.is_unescaped:
+                del d[value]
 
 class HeapCache(object):
     def __init__(self):
         self.reset()
 
-    def reset(self, reset_virtuals=True, trace_branch=True):
-        # contains boxes where the class is already known
-        self.known_class_boxes = {}
+    def reset(self):
+        # maps boxes to values
+        self.values = {}
         # store the boxes that contain newly allocated objects, this maps the
         # boxes to a bool, the bool indicates whether or not the object has
         # escaped the trace or not (True means the box never escaped, False
         # means it did escape), its presences in the mapping shows that it was
         # allocated inside the trace
-        if trace_branch:
-            self.new_boxes = {}
-        else:
-            for box in self.new_boxes:
-                self.new_boxes[box] = False
-        if reset_virtuals:
-            self.likely_virtuals = {}      # only for jit.isvirtual()
+        #if trace_branch:
+            #self.new_boxes = {}
+        #    pass
+        #else:
+            #for box in self.new_boxes:
+            #    self.new_boxes[box] = False
+        #    pass
+        #if reset_virtuals:
+        #    self.likely_virtuals = {}      # only for jit.isvirtual()
         # Tracks which boxes should be marked as escaped when the key box
         # escapes.
-        self.dependencies = {}
-        # contains frame boxes that are not virtualizables
-        if trace_branch:
-            self.nonstandard_virtualizables = {}
+        #self.dependencies = {}
 
         # heap cache
-        # maps descrs to {from_box, to_box} dicts
+        # maps descrs to CacheEntry
         self.heap_cache = {}
         # heap array cache
-        # maps descrs to {index: {from_box: to_box}} dicts
+        # maps descrs to {index: {from_value: to_value}} dicts
         self.heap_array_cache = {}
-        # cache the length of arrays
-        self.length_cache = {}
 
-        # replace_box is called surprisingly often, therefore it's not efficient
-        # to go over all the dicts and fix them.
-        # instead, these two dicts are kept, and a replace_box adds an entry to
-        # each of them.
-        # every time one of the dicts heap_cache, heap_array_cache, length_cache
-        # is accessed, suitable indirections need to be performed
+    def reset_keep_likely_virtuals(self):
+        for value in self.values.itervalues():
+            value.reset_keep_likely_virtual()
+        self.heap_cache = {}
+        self.heap_array_cache = {}
 
-        # this looks all very subtle, but in practice the patterns of
-        # replacements should not be that complex. Usually a box is replaced by
-        # a const, once. Also, if something goes wrong, the effect is that less
-        # caching than possible is done, which is not a huge problem.
-        self.input_indirections = {}
-        self.output_indirections = {}
+    def getvalue(self, box):
+        value = self.values.get(box, None)
+        if not value:
+            value = self.values[box] = HeapCacheValue(box)
+        return value
 
-    def _input_indirection(self, box):
-        return self.input_indirections.get(box, box)
-
-    def _output_indirection(self, box):
-        return self.output_indirections.get(box, box)
+    def getvalues(self, boxes):
+        return [self.getvalue(box) for box in boxes]
 
     def invalidate_caches(self, opnum, descr, argboxes):
         self.mark_escaped(opnum, descr, argboxes)
@@ -64,18 +114,22 @@
     def mark_escaped(self, opnum, descr, argboxes):
         if opnum == rop.SETFIELD_GC:
             assert len(argboxes) == 2
-            box, valuebox = argboxes
-            if self.is_unescaped(box) and self.is_unescaped(valuebox):
-                self.dependencies.setdefault(box, []).append(valuebox)
+            value, fieldvalue = self.getvalues(argboxes)
+            if value.is_unescaped and fieldvalue.is_unescaped:
+                if value.dependencies is None:
+                    value.dependencies = []
+                value.dependencies.append(fieldvalue)
             else:
-                self._escape(valuebox)
+                self._escape(fieldvalue)
         elif opnum == rop.SETARRAYITEM_GC:
             assert len(argboxes) == 3
-            box, indexbox, valuebox = argboxes
-            if self.is_unescaped(box) and self.is_unescaped(valuebox):
-                self.dependencies.setdefault(box, []).append(valuebox)
+            value, indexvalue, fieldvalue = self.getvalues(argboxes)
+            if value.is_unescaped and fieldvalue.is_unescaped:
+                if value.dependencies is None:
+                    value.dependencies = []
+                value.dependencies.append(fieldvalue)
             else:
-                self._escape(valuebox)
+                self._escape(fieldvalue)
         elif (opnum == rop.CALL and
               descr.get_extra_info().oopspecindex == descr.get_extra_info().OS_ARRAYCOPY and
               isinstance(argboxes[3], ConstInt) and
@@ -84,6 +138,7 @@
               len(descr.get_extra_info().write_descrs_arrays) == 1):
             # ARRAYCOPY with constant starts and constant length doesn't escape
             # its argument
+            # XXX really?
             pass
         # GETFIELD_GC, MARK_OPAQUE_PTR, PTR_EQ, and PTR_NE don't escape their
         # arguments
@@ -95,25 +150,20 @@
               opnum != rop.INSTANCE_PTR_EQ and
               opnum != rop.INSTANCE_PTR_NE):
             for box in argboxes:
-                self._escape(box)
+                self._escape_box(box)
 
-    def _escape(self, box):
-        try:
-            unescaped = self.new_boxes[box]
-        except KeyError:
-            pass
-        else:
-            if unescaped:
-                self.new_boxes[box] = False
-        try:
-            del self.likely_virtuals[box]
-        except KeyError:
-            pass
-        try:
-            deps = self.dependencies.pop(box)
-        except KeyError:
-            pass
-        else:
+    def _escape_box(self, box):
+        value = self.values.get(box, None)
+        if not value:
+            return
+        self._escape(value)
+
+    def _escape(self, value):
+        value.is_unescaped = False
+        value.likely_virtual = False
+        deps = value.dependencies
+        value.dependencies = None
+        if deps is not None:
             for dep in deps:
                 self._escape(dep)
 
@@ -146,181 +196,198 @@
             # A special case for ll_arraycopy, because it is so common, and its
             # effects are so well defined.
             elif effectinfo.oopspecindex == effectinfo.OS_ARRAYCOPY:
-                if (
-                    isinstance(argboxes[3], ConstInt) and
-                    isinstance(argboxes[4], ConstInt) and
-                    isinstance(argboxes[5], ConstInt) and
-                    len(effectinfo.write_descrs_arrays) == 1
-                ):
-                    descr = effectinfo.write_descrs_arrays[0]
-                    cache = self.heap_array_cache.get(descr, None)
-                    srcstart = argboxes[3].getint()
-                    dststart = argboxes[4].getint()
-                    length = argboxes[5].getint()
-                    for i in xrange(length):
-                        value = self.getarrayitem(
-                            argboxes[1],
-                            ConstInt(srcstart + i),
-                            descr,
-                        )
-                        if value is not None:
-                            self.setarrayitem(
-                                argboxes[2],
-                                ConstInt(dststart + i),
-                                value,
-                                descr,
-                            )
-                        elif cache is not None:
-                            try:
-                                idx_cache = cache[dststart + i]
-                            except KeyError:
-                                pass
-                            else:
-                                if argboxes[2] in self.new_boxes:
-                                    for frombox in idx_cache.keys():
-                                        if not self.is_unescaped(frombox):
-                                            del idx_cache[frombox]
-                                else:
-                                    idx_cache.clear()
-                    return
-                elif (
-                    argboxes[2] in self.new_boxes and
-                    len(effectinfo.write_descrs_arrays) == 1
-                ):
-                    # Fish the descr out of the effectinfo
-                    cache = self.heap_array_cache.get(effectinfo.write_descrs_arrays[0], None)
-                    if cache is not None:
-                        for idx, cache in cache.iteritems():
-                            for frombox in cache.keys():
-                                if not self.is_unescaped(frombox):
-                                    del cache[frombox]
-                    return
+                self._clear_caches_arraycopy(opnum, descr, argboxes, effectinfo)
+                return
             else:
-                # Only invalidate things that are either escaped or arguments
-                for descr, boxes in self.heap_cache.iteritems():
-                    for box in boxes.keys():
-                        if not self.is_unescaped(box) or box in argboxes:
-                            del boxes[box]
+                # first escape arguments:
+                for argbox in argboxes:
+                    self._escape_box(argbox)
+
+                # Only invalidate things that are escaped
+                # XXX can do better, only do it for the descrs in the effectinfo
+                for descr, cache in self.heap_cache.iteritems():
+                    cache.invalidate_unescaped()
                 for descr, indices in self.heap_array_cache.iteritems():
-                    for boxes in indices.itervalues():
-                        for box in boxes.keys():
-                            if not self.is_unescaped(box) or box in argboxes:
-                                del boxes[box]
+                    for cache in indices.itervalues():
+                        cache.invalidate_unescaped()
                 return
 
         # XXX not completely sure, but I *think* it is needed to reset() the
         # state at least in the 'CALL_*' operations that release the GIL.  We
         # tried to do only the kind of resetting done by the two loops just
         # above, but hit an assertion in "pypy test_multiprocessing.py".
-        self.reset(reset_virtuals=False, trace_branch=False)
+        self.reset_keep_likely_virtuals()
+
+    def _clear_caches_arraycopy(self, opnum, desrc, argboxes, effectinfo):
+        seen_allocation_of_target = self.getvalue(argboxes[2]).seen_allocation
+        if (
+            isinstance(argboxes[3], ConstInt) and
+            isinstance(argboxes[4], ConstInt) and
+            isinstance(argboxes[5], ConstInt) and
+            len(effectinfo.write_descrs_arrays) == 1
+        ):
+            descr = effectinfo.write_descrs_arrays[0]
+            cache = self.heap_array_cache.get(descr, None)
+            srcstart = argboxes[3].getint()
+            dststart = argboxes[4].getint()
+            length = argboxes[5].getint()
+            for i in xrange(length):
+                value = self.getarrayitem(
+                    argboxes[1],
+                    ConstInt(srcstart + i),
+                    descr,
+                )
+                if value is not None:
+                    self.setarrayitem(
+                        argboxes[2],
+                        ConstInt(dststart + i),
+                        value,
+                        descr,
+                    )
+                elif cache is not None:
+                    try:
+                        idx_cache = cache[dststart + i]
+                    except KeyError:
+                        pass
+                    else:
+                        idx_cache._clear_cache_on_write(seen_allocation_of_target)
+            return
+        elif (
+            len(effectinfo.write_descrs_arrays) == 1
+        ):
+            # Fish the descr out of the effectinfo
+            cache = self.heap_array_cache.get(effectinfo.write_descrs_arrays[0], None)
+            if cache is not None:
+                for idx, cache in cache.iteritems():
+                    cache._clear_cache_on_write(seen_allocation_of_target)
+            return
+        self.reset_keep_likely_virtuals()
 
     def is_class_known(self, box):
-        return box in self.known_class_boxes
+        value = self.values.get(box, None)
+        if value:
+            return value.known_class
+        return False
 
     def class_now_known(self, box):
-        self.known_class_boxes[box] = None
+        self.getvalue(box).known_class = True
 
     def is_nonstandard_virtualizable(self, box):
-        return box in self.nonstandard_virtualizables
+        value = self.values.get(box, None)
+        if value:
+            return value.nonstandard_virtualizable
+        return False
 
     def nonstandard_virtualizables_now_known(self, box):
-        self.nonstandard_virtualizables[box] = None
+        self.getvalue(box).nonstandard_virtualizable = True
 
     def is_unescaped(self, box):
-        return self.new_boxes.get(box, False)
+        value = self.values.get(box, None)
+        if value:
+            return value.is_unescaped
+        return False
 
     def is_likely_virtual(self, box):
-        return box in self.likely_virtuals
+        value = self.values.get(box, None)
+        if value:
+            return value.likely_virtual
+        return False
 
     def new(self, box):
-        self.new_boxes[box] = True
-        self.likely_virtuals[box] = None
+        value = self.getvalue(box)
+        value.is_unescaped = True
+        value.likely_virtual = True
+        value.seen_allocation = True
 
     def new_array(self, box, lengthbox):
         self.new(box)
         self.arraylen_now_known(box, lengthbox)
 
     def getfield(self, box, descr):
-        box = self._input_indirection(box)
-        d = self.heap_cache.get(descr, None)
-        if d:
-            tobox = d.get(box, None)
-            return self._output_indirection(tobox)
+        value = self.values.get(box, None)
+        if value:
+            cache = self.heap_cache.get(descr, None)
+            if cache:
+                tovalue = cache.read(value)
+                if tovalue:
+                    return tovalue.box
         return None
 
     def getfield_now_known(self, box, descr, fieldbox):
-        box = self._input_indirection(box)
-        fieldbox = self._input_indirection(fieldbox)
-        self.heap_cache.setdefault(descr, {})[box] = fieldbox
+        value = self.getvalue(box)
+        fieldvalue = self.getvalue(fieldbox)
+        cache = self.heap_cache.get(descr, None)
+        if cache is None:
+            cache = self.heap_cache[descr] = CacheEntry()
+        cache.read_now_known(value, fieldvalue)
 
     def setfield(self, box, fieldbox, descr):
-        d = self.heap_cache.get(descr, None)
-        new_d = self._do_write_with_aliasing(d, box, fieldbox)
-        self.heap_cache[descr] = new_d
-
-    def _do_write_with_aliasing(self, d, box, fieldbox):
-        box = self._input_indirection(box)
-        fieldbox = self._input_indirection(fieldbox)
-        # slightly subtle logic here
-        # a write to an arbitrary box, all other boxes can alias this one
-        if not d or box not in self.new_boxes:
-            # therefore we throw away the cache
-            return {box: fieldbox}
-        # the object we are writing to is freshly allocated
-        # only remove some boxes from the cache
-        new_d = {}
-        for frombox, tobox in d.iteritems():
-            # the other box is *also* freshly allocated
-            # therefore frombox and box *must* contain different objects
-            # thus we can keep it in the cache
-            if frombox in self.new_boxes:
-                new_d[frombox] = tobox
-        new_d[box] = fieldbox
-        return new_d
+        cache = self.heap_cache.get(descr, None)
+        if cache is None:
+            cache = self.heap_cache[descr] = CacheEntry()
+        value = self.getvalue(box)
+        fieldvalue = self.getvalue(fieldbox)
+        cache.do_write_with_aliasing(value, fieldvalue)
 
     def getarrayitem(self, box, indexbox, descr):
         if not isinstance(indexbox, ConstInt):
-            return
-        box = self._input_indirection(box)
+            return None
+        value = self.values.get(box, None)
+        if value is None:
+            return None
         index = indexbox.getint()
         cache = self.heap_array_cache.get(descr, None)
         if cache:
             indexcache = cache.get(index, None)
             if indexcache is not None:
-                return self._output_indirection(indexcache.get(box, None))
+                resvalue = indexcache.read(value)
+                if resvalue:
+                    return resvalue.box
+        return None
 
-    def getarrayitem_now_known(self, box, indexbox, valuebox, descr):
+    def _get_or_make_array_cache_entry(self, indexbox, descr):
         if not isinstance(indexbox, ConstInt):
-            return
-        box = self._input_indirection(box)
-        valuebox = self._input_indirection(valuebox)
+            return None
         index = indexbox.getint()
         cache = self.heap_array_cache.setdefault(descr, {})
         indexcache = cache.get(index, None)
-        if indexcache is not None:
-            indexcache[box] = valuebox
-        else:
-            cache[index] = {box: valuebox}
+        if indexcache is None:
+            cache[index] = indexcache = CacheEntry()
+        return indexcache
 
-    def setarrayitem(self, box, indexbox, valuebox, descr):
+
+    def getarrayitem_now_known(self, box, indexbox, fieldbox, descr):
+        value = self.getvalue(box)
+        fieldvalue = self.getvalue(fieldbox)
+        indexcache = self._get_or_make_array_cache_entry(indexbox, descr)
+        if indexcache:
+            indexcache.read_now_known(value, fieldvalue)
+
+    def setarrayitem(self, box, indexbox, fieldbox, descr):
         if not isinstance(indexbox, ConstInt):
             cache = self.heap_array_cache.get(descr, None)
             if cache is not None:
                 cache.clear()
             return
-        index = indexbox.getint()
-        cache = self.heap_array_cache.setdefault(descr, {})

From noreply at buildbot.pypy.org  Wed Apr 22 12:20:47 2015
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Wed, 22 Apr 2015 12:20:47 +0200 (CEST)
Subject: [pypy-commit] pypy default: Merged pypy/pypy into default
Message-ID: <20150422102047.060621C06CD@cobra.cs.uni-duesseldorf.de>

Author: anton_gulenko 
Branch: 
Changeset: r76870:fe4c27d34d37
Date: 2015-03-07 16:20 +0100
http://bitbucket.org/pypy/pypy/changeset/fe4c27d34d37/

Log:	Merged pypy/pypy into default

diff too long, truncating to 2000 out of 4777 lines

diff --git a/lib_pypy/cffi.egg-info b/lib_pypy/cffi.egg-info
--- a/lib_pypy/cffi.egg-info
+++ b/lib_pypy/cffi.egg-info
@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: cffi
-Version: 0.8.6+
+Version: 0.9.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "0.8.6+"
-__version_info__ = (0, 8, 6, "plus")
+__version__ = "0.9.0"
+__version_info__ = (0, 9, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/gdbm.py b/lib_pypy/gdbm.py
--- a/lib_pypy/gdbm.py
+++ b/lib_pypy/gdbm.py
@@ -20,9 +20,11 @@
 } datum;
 
 datum gdbm_fetch(void*, datum);
+datum pygdbm_fetch(void*, char*, int);
 int gdbm_delete(void*, datum);
 int gdbm_store(void*, datum, datum, int);
 int gdbm_exists(void*, datum);
+int pygdbm_exists(void*, char*, int);
 
 int gdbm_reorganize(void*);
 
@@ -37,19 +39,29 @@
 ''')
 
 try:
+    verify_code = '''
+    #include "gdbm.h"
+
+    static datum pygdbm_fetch(GDBM_FILE gdbm_file, char *dptr, int dsize) {
+        datum key = {dptr, dsize};
+        return gdbm_fetch(gdbm_file, key);
+    }
+
+    static int pygdbm_exists(GDBM_FILE gdbm_file, char *dptr, int dsize) {
+        datum key = {dptr, dsize};
+        return gdbm_exists(gdbm_file, key);
+    }
+    
+    '''
     if sys.platform.startswith('freebsd'):
         import os.path
         _localbase = os.environ.get('LOCALBASE', '/usr/local')
-        lib = ffi.verify('''
-        #include "gdbm.h"
-        ''', libraries=['gdbm'],
+        lib = ffi.verify(verify_code, libraries=['gdbm'],
              include_dirs=[os.path.join(_localbase, 'include')],
              library_dirs=[os.path.join(_localbase, 'lib')]
         )
     else:
-        lib = ffi.verify('''
-        #include "gdbm.h"
-        ''', libraries=['gdbm'])
+        lib = ffi.verify(verify_code, libraries=['gdbm'])
 except cffi.VerificationError as e:
     # distutils does not preserve the actual message,
     # but the verification is simple enough that the
@@ -59,6 +71,13 @@
 class error(Exception):
     pass
 
+def _checkstr(key):
+    if isinstance(key, unicode):
+        key = key.encode("ascii")
+    if not isinstance(key, str):
+        raise TypeError("gdbm mappings have string indices only")
+    return key
+
 def _fromstr(key):
     if isinstance(key, unicode):
         key = key.encode("ascii")
@@ -107,12 +126,14 @@
 
     def __contains__(self, key):
         self._check_closed()
-        return lib.gdbm_exists(self.ll_dbm, _fromstr(key))
+        key = _checkstr(key)
+        return lib.pygdbm_exists(self.ll_dbm, key, len(key))
     has_key = __contains__
 
     def __getitem__(self, key):
         self._check_closed()
-        drec = lib.gdbm_fetch(self.ll_dbm, _fromstr(key))
+        key = _checkstr(key)        
+        drec = lib.pygdbm_fetch(self.ll_dbm, key, len(key))
         if not drec.dptr:
             raise KeyError(key)
         res = str(ffi.buffer(drec.dptr, drec.dsize))
diff --git a/lib_pypy/pyrepl/readline.py b/lib_pypy/pyrepl/readline.py
--- a/lib_pypy/pyrepl/readline.py
+++ b/lib_pypy/pyrepl/readline.py
@@ -73,7 +73,6 @@
     assume_immutable_completions = False
     use_brackets = False
     sort_in_column = True
-    tab_insert_spaces_if_stem_is_empty = False
 
     def error(self, msg="none"):
         pass    # don't show error messages by default
@@ -87,7 +86,7 @@
         return ''.join(b[p+1:self.pos])
 
     def get_completions(self, stem):
-        if len(stem) == 0 and self.tab_insert_spaces_if_stem_is_empty:
+        if len(stem) == 0 and self.more_lines is not None:
             b = self.buffer
             p = self.pos
             while p > 0 and b[p - 1] != '\n':
@@ -141,12 +140,16 @@
 
     def collect_keymap(self):
         return super(ReadlineAlikeReader, self).collect_keymap() + (
-            (r'\n', 'maybe-accept'),)
+            (r'\n', 'maybe-accept'),
+            (r'\', 'backspace-dedent'),
+            )
 
     def __init__(self, console):
         super(ReadlineAlikeReader, self).__init__(console)
         self.commands['maybe_accept'] = maybe_accept
         self.commands['maybe-accept'] = maybe_accept
+        self.commands['backspace_dedent'] = backspace_dedent
+        self.commands['backspace-dedent'] = backspace_dedent
 
     def after_command(self, cmd):
         super(ReadlineAlikeReader, self).after_command(cmd)
@@ -164,6 +167,28 @@
                 if self.pos > len(self.buffer):
                     self.pos = len(self.buffer)
 
+def _get_this_line_indent(buffer, pos):
+    indent = 0
+    while pos > 0 and buffer[pos - 1] in " \t":
+        indent += 1
+        pos -= 1
+    if pos > 0 and buffer[pos - 1] == "\n":
+        return indent
+    return 0
+
+def _get_previous_line_indent(buffer, pos):
+    prevlinestart = pos
+    while prevlinestart > 0 and buffer[prevlinestart - 1] != "\n":
+        prevlinestart -= 1
+    prevlinetext = prevlinestart
+    while prevlinetext < pos and buffer[prevlinetext] in " \t":
+        prevlinetext += 1
+    if prevlinetext == pos:
+        indent = None
+    else:
+        indent = prevlinetext - prevlinestart
+    return prevlinestart, indent
+
 class maybe_accept(commands.Command):
     def do(self):
         r = self.reader
@@ -172,13 +197,39 @@
         # if there are already several lines and the cursor
         # is not on the last one, always insert a new \n.
         text = r.get_unicode()
-        if "\n" in r.buffer[r.pos:]:
+        if ("\n" in r.buffer[r.pos:] or
+            (r.more_lines is not None and r.more_lines(text))):
+            #
+            # auto-indent the next line like the previous line
+            prevlinestart, indent = _get_previous_line_indent(r.buffer, r.pos)
             r.insert("\n")
-        elif r.more_lines is not None and r.more_lines(text):
-            r.insert("\n")
+            if indent:
+                for i in range(prevlinestart, prevlinestart + indent):
+                    r.insert(r.buffer[i])
         else:
             self.finish = 1
 
+class backspace_dedent(commands.Command):
+    def do(self):
+        r = self.reader
+        b = r.buffer
+        if r.pos > 0:
+            repeat = 1
+            if b[r.pos - 1] != "\n":
+                indent = _get_this_line_indent(b, r.pos)
+                if indent > 0:
+                    ls = r.pos - indent
+                    while ls > 0:
+                        ls, pi = _get_previous_line_indent(b, ls - 1)
+                        if pi is not None and pi < indent:
+                            repeat = indent - pi
+                            break
+            r.pos -= repeat
+            del b[r.pos:r.pos + repeat]
+            r.dirty = 1
+        else:
+            self.reader.error("can't backspace at start")
+
 # ____________________________________________________________
 
 class _ReadlineWrapper(object):
@@ -212,15 +263,14 @@
         boolean value is true.
         """
         reader = self.get_reader()
-        saved = reader.more_lines, reader.tab_insert_spaces_if_stem_is_empty
+        saved = reader.more_lines
         try:
             reader.more_lines = more_lines
             reader.ps1 = reader.ps2 = ps1
             reader.ps3 = reader.ps4 = ps2
-            reader.tab_insert_spaces_if_stem_is_empty = True
             return reader.readline(returns_unicode=returns_unicode)
         finally:
-            reader.more_lines, reader.tab_insert_spaces_if_stem_is_empty = saved
+            reader.more_lines = saved
 
     def parse_and_bind(self, string):
         pass  # XXX we don't support parsing GNU-readline-style init files
diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -36,7 +36,7 @@
 def pytest_addoption(parser):
     from rpython.conftest import pytest_addoption
     pytest_addoption(parser)
-    
+
     group = parser.getgroup("pypy options")
     group.addoption('-A', '--runappdirect', action="store_true",
            default=False, dest="runappdirect",
@@ -44,6 +44,9 @@
     group.addoption('--direct', action="store_true",
            default=False, dest="rundirect",
            help="run pexpect tests directly")
+    group.addoption('--raise-operr', action="store_true",
+            default=False, dest="raise_operr",
+            help="Show the interp-level OperationError in app-level tests")
 
 def pytest_funcarg__space(request):
     from pypy.tool.pytest.objspace import gettestobjspace
diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -36,7 +36,8 @@
    "PyPy home directory".  The arguments are:
 
    * ``home``: NULL terminated path to an executable inside the pypy directory
-     (can be a .so name, can be made up)
+     (can be a .so name, can be made up).  Used to look up the standard
+     library, and is also set as ``sys.executable``.
 
    * ``verbose``: if non-zero, it will print error messages to stderr
 
diff --git a/pypy/doc/jit-hooks.rst b/pypy/doc/jit-hooks.rst
--- a/pypy/doc/jit-hooks.rst
+++ b/pypy/doc/jit-hooks.rst
@@ -39,3 +39,30 @@
     Reason is a string, the meaning of other arguments is the same
     as attributes on JitLoopInfo object
 
+.. function:: enable_debug()
+
+    Start recording debugging counters for ``get_stats_snapshot``
+
+.. function:: disable_debug()
+
+    Stop recording debugging counters for ``get_stats_snapshot``
+
+.. function:: get_stats_snapshot()
+
+    Get the jit status in the specific moment in time. Note that this
+    is eager - the attribute access is not lazy, if you need new stats
+    you need to call this function again. You might want to call
+    ``enable_debug`` to get more information. It returns an instance
+    of ``JitInfoSnapshot``
+
+.. class:: JitInfoSnapshot
+
+    A class describing current snapshot. Usable attributes:
+
+    * ``counters`` - internal JIT integer counters
+
+    * ``counter_times`` - internal JIT float counters, notably time spent
+      TRACING and in the JIT BACKEND
+
+    * ``loop_run_times`` - counters for number of times loops are run, only
+      works when ``enable_debug`` is called.
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -30,3 +30,10 @@
 .. branch: alt_errno
 Add an alternative location to save LastError, errno around ctypes,
 cffi external calls so things like pdb will not overwrite it
+
+.. branch: nonquadratic-heapcache
+Speed up the warmup times of the JIT by removing a quadratic algorithm in the
+heapcache.
+
+.. branch: online-transforms-2
+Simplify flow graphs on the fly during annotation phase.
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -106,6 +106,9 @@
         space.call_function(w_pathsetter, w_path)
         # import site
         try:
+            space.setattr(space.getbuiltinmodule('sys'),
+                          space.wrap('executable'),
+                          space.wrap(home))
             import_ = space.getattr(space.getbuiltinmodule('__builtin__'),
                                     space.wrap('__import__'))
             space.call_function(import_, space.wrap('site'))
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -1,5 +1,5 @@
 #! /usr/bin/env python
-# App-level version of py.py.
+# This is pure Python code that handles the main entry point into "pypy".
 # See test/test_app_main.
 
 # Missing vs CPython: -d, -t, -v, -x, -3
@@ -157,10 +157,13 @@
             current = group
     raise SystemExit
 
+def get_sys_executable():
+    return getattr(sys, 'executable', 'pypy')
+
 def print_help(*args):
     import os
     print 'usage: %s [option] ... [-c cmd | -m mod | file | -] [arg] ...' % (
-        sys.executable,)
+        get_sys_executable(),)
     print USAGE1,
     if 'pypyjit' in sys.builtin_module_names:
         print "--jit options: advanced JIT options: try 'off' or 'help'"
@@ -171,7 +174,7 @@
     try:
         import pypyjit
     except ImportError:
-        print >> sys.stderr, "No jit support in %s" % (sys.executable,)
+        print >> sys.stderr, "No jit support in %s" % (get_sys_executable(),)
         return
     items = sorted(pypyjit.defaults.items())
     print 'Advanced JIT options: a comma-separated list of OPTION=VALUE:'
@@ -209,7 +212,7 @@
         raise SystemExit
     if 'pypyjit' not in sys.builtin_module_names:
         print >> sys.stderr, ("Warning: No jit support in %s" %
-                              (sys.executable,))
+                              (get_sys_executable(),))
     else:
         import pypyjit
         pypyjit.set_param(jitparam)
@@ -219,8 +222,8 @@
 
 def print_error(msg):
     print >> sys.stderr, msg
-    print >> sys.stderr, 'usage: %s [options]' % (sys.executable,)
-    print >> sys.stderr, 'Try `%s -h` for more information.' % (sys.executable,)
+    print >> sys.stderr, 'usage: %s [options]' % (get_sys_executable(),)
+    print >> sys.stderr, 'Try `%s -h` for more information.' % (get_sys_executable(),)
 
 def fdopen(fd, mode, bufsize=-1):
     try:
@@ -514,6 +517,10 @@
     elif not sys.stdout.isatty():
         set_fully_buffered_io()
 
+    if we_are_translated():
+        import __pypy__
+        __pypy__.save_module_content_for_future_reload(sys)
+
     mainmodule = type(sys)('__main__')
     sys.modules['__main__'] = mainmodule
 
diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py
--- a/pypy/interpreter/mixedmodule.py
+++ b/pypy/interpreter/mixedmodule.py
@@ -55,7 +55,10 @@
         if self.w_initialdict is None:
             Module.init(self, space)
             if not self.lazy and self.w_initialdict is None:
-                self.w_initialdict = space.call_method(self.w_dict, 'items')
+                self.save_module_content_for_future_reload()
+
+    def save_module_content_for_future_reload(self):
+        self.w_initialdict = self.space.call_method(self.w_dict, 'items')
 
 
     def get_applevel_name(cls):
@@ -119,7 +122,7 @@
                 w_value = self.get(name)
                 space.setitem(self.w_dict, space.new_interned_str(name), w_value)
             self.lazy = False
-            self.w_initialdict = space.call_method(self.w_dict, 'items')
+            self.save_module_content_for_future_reload()
         return self.w_dict
 
     def _cleanup_(self):
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -373,7 +373,7 @@
         config = make_config(None)
         space = make_objspace(config)
         w_executable = space.wrap('executable')
-        assert space.str_w(space.getattr(space.sys, w_executable)) == 'py.py'
+        assert space.findattr(space.sys, w_executable) is None
         space.setattr(space.sys, w_executable, space.wrap('foobar'))
         assert space.str_w(space.getattr(space.sys, w_executable)) == 'foobar'
         space.startup()
diff --git a/pypy/interpreter/test/test_targetpypy.py b/pypy/interpreter/test/test_targetpypy.py
--- a/pypy/interpreter/test/test_targetpypy.py
+++ b/pypy/interpreter/test/test_targetpypy.py
@@ -8,7 +8,7 @@
         entry_point = get_entry_point(config)[0]
         entry_point(['pypy-c' , '-S', '-c', 'print 3'])
 
-def test_exeucte_source(space):
+def test_execute_source(space):
     _, d = create_entry_point(space, None)
     execute_source = d['pypy_execute_source']
     lls = rffi.str2charp("import sys; sys.modules['xyz'] = 3")
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -82,6 +82,8 @@
         'strategy'                  : 'interp_magic.strategy',  # dict,set,list
         'set_debug'                 : 'interp_magic.set_debug',
         'locals_to_fast'            : 'interp_magic.locals_to_fast',
+        'save_module_content_for_future_reload':
+                          'interp_magic.save_module_content_for_future_reload',
     }
     if sys.platform == 'win32':
         interpleveldefs['get_console_cp'] = 'interp_magic.get_console_cp'
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -1,6 +1,7 @@
 from pypy.interpreter.error import OperationError, wrap_oserror
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.pyframe import PyFrame
+from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rlib.objectmodel import we_are_translated
 from pypy.objspace.std.dictmultiobject import W_DictMultiObject
 from pypy.objspace.std.listobject import W_ListObject
@@ -130,3 +131,7 @@
 def locals_to_fast(space, w_frame):
     assert isinstance(w_frame, PyFrame)
     w_frame.locals2fast()
+
+ at unwrap_spec(w_module=MixedModule)
+def save_module_content_for_future_reload(space, w_module):
+    w_module.save_module_content_for_future_reload()
diff --git a/pypy/module/__pypy__/test/test_magic.py b/pypy/module/__pypy__/test/test_magic.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__pypy__/test/test_magic.py
@@ -0,0 +1,15 @@
+
+class AppTestMagic:
+    spaceconfig = dict(usemodules=['__pypy__'])
+
+    def test_save_module_content_for_future_reload(self):
+        import sys, __pypy__
+        d = sys.dont_write_bytecode
+        sys.dont_write_bytecode = "hello world"
+        __pypy__.save_module_content_for_future_reload(sys)
+        sys.dont_write_bytecode = d
+        reload(sys)
+        assert sys.dont_write_bytecode == "hello world"
+        #
+        sys.dont_write_bytecode = d
+        __pypy__.save_module_content_for_future_reload(sys)
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -8,7 +8,7 @@
     appleveldefs = {
         }
     interpleveldefs = {
-        '__version__': 'space.wrap("0.8.6+")',
+        '__version__': 'space.wrap("0.9.0")',
 
         'load_library': 'libraryobj.load_library',
 
diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -81,4 +81,5 @@
     if size < 0:
         raise oefmt(space.w_TypeError,
                     "don't know the size pointed to by '%s'", ctype.name)
-    return space.wrap(MiniBuffer(LLBuffer(w_cdata._cdata, size), w_cdata))
+    ptr = w_cdata.unsafe_escaping_ptr()    # w_cdata kept alive by MiniBuffer()
+    return space.wrap(MiniBuffer(LLBuffer(ptr, size), w_cdata))
diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py
--- a/pypy/module/_cffi_backend/ccallback.py
+++ b/pypy/module/_cffi_backend/ccallback.py
@@ -48,9 +48,12 @@
             raise oefmt(space.w_NotImplementedError,
                         "%s: callback with unsupported argument or "
                         "return type or with '...'", self.getfunctype().name)
-        res = clibffi.c_ffi_prep_closure(self.get_closure(), cif_descr.cif,
-                                         invoke_callback,
-                                         rffi.cast(rffi.VOIDP, self.unique_id))
+        with self as ptr:
+            closure_ptr = rffi.cast(clibffi.FFI_CLOSUREP, ptr)
+            unique_id = rffi.cast(rffi.VOIDP, self.unique_id)
+            res = clibffi.c_ffi_prep_closure(closure_ptr, cif_descr.cif,
+                                             invoke_callback,
+                                             unique_id)
         if rffi.cast(lltype.Signed, res) != clibffi.FFI_OK:
             raise OperationError(space.w_SystemError,
                 space.wrap("libffi failed to build this callback"))
@@ -62,12 +65,9 @@
             from pypy.module.thread.os_thread import setup_threads
             setup_threads(space)
 
-    def get_closure(self):
-        return rffi.cast(clibffi.FFI_CLOSUREP, self._cdata)
-
     #@rgc.must_be_light_finalizer
     def __del__(self):
-        clibffi.closureHeap.free(self.get_closure())
+        clibffi.closureHeap.free(rffi.cast(clibffi.FFI_CLOSUREP, self._ptr))
         if self.ll_error:
             lltype.free(self.ll_error, flavor='raw')
 
@@ -106,7 +106,7 @@
         fresult = self.getfunctype().ctitem
         if fresult.size > 0:
             misc._raw_memcopy(self.ll_error, ll_res, fresult.size)
-            keepalive_until_here(self)
+            keepalive_until_here(self)     # to keep self.ll_error alive
 
 
 global_callback_mapping = rweakref.RWeakValueDictionary(int, W_CDataCallback)
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -14,21 +14,37 @@
 
 
 class W_CData(W_Root):
-    _attrs_ = ['space', '_cdata', 'ctype', '_lifeline_']
-    _immutable_fields_ = ['_cdata', 'ctype']
-    _cdata = lltype.nullptr(rffi.CCHARP.TO)
+    _attrs_ = ['space', '_ptr', 'ctype', '_lifeline_']
+    _immutable_fields_ = ['_ptr', 'ctype']
+    _ptr = lltype.nullptr(rffi.CCHARP.TO)
 
-    def __init__(self, space, cdata, ctype):
+    def __init__(self, space, ptr, ctype):
         from pypy.module._cffi_backend import ctypeobj
-        assert lltype.typeOf(cdata) == rffi.CCHARP
+        assert lltype.typeOf(ptr) == rffi.CCHARP
         assert isinstance(ctype, ctypeobj.W_CType)
         self.space = space
-        self._cdata = cdata    # don't forget keepalive_until_here!
+        self._ptr = ptr    # don't access directly!  use "with cdata as ptr:"
         self.ctype = ctype
 
+    def __enter__(self):
+        """Use 'with cdata as ptr:' to access the raw memory.  It will
+        stay alive at least until the end of the 'with' block.
+        """
+        return self._ptr
+
+    def __exit__(self, *args):
+        keepalive_until_here(self)
+
+    def unsafe_escaping_ptr(self):
+        """Generally unsafe: escape the pointer to raw memory.
+        If 'self' is a subclass that frees the pointer in a destructor,
+        it may be freed under your feet at any time.
+        """
+        return self._ptr
+
     def _repr_extra(self):
-        extra = self.ctype.extra_repr(self._cdata)
-        keepalive_until_here(self)
+        with self as ptr:
+            extra = self.ctype.extra_repr(ptr)
         return extra
 
     def _repr_extra_owning(self):
@@ -54,11 +70,13 @@
             self.ctype.name, extra1, extra2))
 
     def nonzero(self):
-        return self.space.wrap(bool(self._cdata))
+        with self as ptr:
+            nonzero = bool(ptr)
+        return self.space.wrap(nonzero)
 
     def int(self, space):
-        w_result = self.ctype.cast_to_int(self._cdata)
-        keepalive_until_here(self)
+        with self as ptr:
+            w_result = self.ctype.cast_to_int(ptr)
         return w_result
 
     def long(self, space):
@@ -69,8 +87,8 @@
         return w_result
 
     def float(self):
-        w_result = self.ctype.float(self._cdata)
-        keepalive_until_here(self)
+        with self as ptr:
+            w_result = self.ctype.float(ptr)
         return w_result
 
     def len(self):
@@ -88,20 +106,19 @@
         def _cmp(self, w_other):
             from pypy.module._cffi_backend.ctypeprim import W_CTypePrimitive
             space = self.space
-            cdata1 = self._cdata
-            if isinstance(w_other, W_CData):
-                cdata2 = w_other._cdata
-            else:
+            if not isinstance(w_other, W_CData):
                 return space.w_NotImplemented
 
-            if requires_ordering:
-                if (isinstance(self.ctype, W_CTypePrimitive) or
-                    isinstance(w_other.ctype, W_CTypePrimitive)):
-                    raise OperationError(space.w_TypeError,
-                       space.wrap("cannot do comparison on a primitive cdata"))
-                cdata1 = rffi.cast(lltype.Unsigned, cdata1)
-                cdata2 = rffi.cast(lltype.Unsigned, cdata2)
-            return space.newbool(op(cdata1, cdata2))
+            with self as ptr1, w_other as ptr2:
+                if requires_ordering:
+                    if (isinstance(self.ctype, W_CTypePrimitive) or
+                        isinstance(w_other.ctype, W_CTypePrimitive)):
+                        raise OperationError(space.w_TypeError, space.wrap(
+                            "cannot do comparison on a primitive cdata"))
+                    ptr1 = rffi.cast(lltype.Unsigned, ptr1)
+                    ptr2 = rffi.cast(lltype.Unsigned, ptr2)
+                result = op(ptr1, ptr2)
+            return space.newbool(result)
         #
         return func_with_new_name(_cmp, name)
 
@@ -113,7 +130,8 @@
     ge = _make_comparison('ge')
 
     def hash(self):
-        h = rffi.cast(lltype.Signed, self._cdata)
+        ptr = self.unsafe_escaping_ptr()
+        h = rffi.cast(lltype.Signed, ptr)
         # To hash pointers in dictionaries.  Assumes that h shows some
         # alignment (to 4, 8, maybe 16 bytes), so we use the following
         # formula to avoid the trailing bits being always 0.
@@ -128,26 +146,27 @@
             i = space.getindex_w(w_index, space.w_IndexError)
             ctype = self.ctype._check_subscript_index(self, i)
             w_o = self._do_getitem(ctype, i)
-        keepalive_until_here(self)
         return w_o
 
     def _do_getitem(self, ctype, i):
         ctitem = ctype.ctitem
-        return ctitem.convert_to_object(
-            rffi.ptradd(self._cdata, i * ctitem.size))
+        with self as ptr:
+            return ctitem.convert_to_object(
+                rffi.ptradd(ptr, i * ctitem.size))
 
     def setitem(self, w_index, w_value):
         space = self.space
         if space.isinstance_w(w_index, space.w_slice):
-            self._do_setslice(w_index, w_value)
+            with self as ptr:
+                self._do_setslice(w_index, w_value, ptr)
         else:
             i = space.getindex_w(w_index, space.w_IndexError)
             ctype = self.ctype._check_subscript_index(self, i)
             ctitem = ctype.ctitem
-            ctitem.convert_from_object(
-                rffi.ptradd(self._cdata, i * ctitem.size),
-                w_value)
-        keepalive_until_here(self)
+            with self as ptr:
+                ctitem.convert_from_object(
+                    rffi.ptradd(ptr, i * ctitem.size),
+                    w_value)
 
     def _do_getslicearg(self, w_slice):
         from pypy.module._cffi_backend.ctypeptr import W_CTypePointer
@@ -188,14 +207,15 @@
             ctarray = newtype.new_array_type(space, ctptr, space.w_None)
             ctptr.cache_array_type = ctarray
         #
-        p = rffi.ptradd(self._cdata, start * ctarray.ctitem.size)
-        return W_CDataSliced(space, p, ctarray, length)
+        ptr = self.unsafe_escaping_ptr()
+        ptr = rffi.ptradd(ptr, start * ctarray.ctitem.size)
+        return W_CDataSliced(space, ptr, ctarray, length)
 
-    def _do_setslice(self, w_slice, w_value):
+    def _do_setslice(self, w_slice, w_value, ptr):
         ctptr, start, length = self._do_getslicearg(w_slice)
         ctitem = ctptr.ctitem
         ctitemsize = ctitem.size
-        cdata = rffi.ptradd(self._cdata, start * ctitemsize)
+        target = rffi.ptradd(ptr, start * ctitemsize)
         #
         if isinstance(w_value, W_CData):
             from pypy.module._cffi_backend import ctypearray
@@ -204,9 +224,8 @@
                 ctv.ctitem is ctitem and
                 w_value.get_array_length() == length):
                 # fast path: copying from exactly the correct type
-                s = w_value._cdata
-                rffi.c_memcpy(cdata, s, ctitemsize * length)
-                keepalive_until_here(w_value)
+                with w_value as source:
+                    rffi.c_memcpy(target, source, ctitemsize * length)
                 return
         #
         # A fast path for [0:N] = "somestring".
@@ -221,7 +240,7 @@
                 raise oefmt(space.w_ValueError,
                             "need a string of length %d, got %d",
                             length, len(value))
-            copy_string_to_raw(llstr(value), cdata, 0, length)
+            copy_string_to_raw(llstr(value), target, 0, length)
             return
         #
         w_iter = space.iter(w_value)
@@ -233,8 +252,8 @@
                     raise
                 raise oefmt(space.w_ValueError,
                             "need %d values to unpack, got %d", length, i)
-            ctitem.convert_from_object(cdata, w_item)
-            cdata = rffi.ptradd(cdata, ctitemsize)
+            ctitem.convert_from_object(target, w_item)
+            target = rffi.ptradd(target, ctitemsize)
         try:
             space.next(w_iter)
         except OperationError, e:
@@ -247,7 +266,8 @@
     def _add_or_sub(self, w_other, sign):
         space = self.space
         i = sign * space.getindex_w(w_other, space.w_OverflowError)
-        return self.ctype.add(self._cdata, i)
+        ptr = self.unsafe_escaping_ptr()
+        return self.ctype.add(ptr, i)
 
     def add(self, w_other):
         return self._add_or_sub(w_other, +1)
@@ -268,9 +288,11 @@
                             self.ctype.name, ct.name)
             #
             itemsize = ct.ctitem.size
-            if itemsize <= 0: itemsize = 1
-            diff = (rffi.cast(lltype.Signed, self._cdata) -
-                    rffi.cast(lltype.Signed, w_other._cdata)) // itemsize
+            if itemsize <= 0:
+                itemsize = 1
+            with self as ptr1, w_other as ptr2:
+                diff = (rffi.cast(lltype.Signed, ptr1) -
+                        rffi.cast(lltype.Signed, ptr2)) // itemsize
             return space.wrap(diff)
         #
         return self._add_or_sub(w_other, -1)
@@ -279,17 +301,19 @@
         return self.ctype.getcfield(self.space.str_w(w_attr))
 
     def getattr(self, w_attr):
-        w_res = self.getcfield(w_attr).read(self._cdata)
-        keepalive_until_here(self)
+        cfield = self.getcfield(w_attr)
+        with self as ptr:
+            w_res = cfield.read(ptr)
         return w_res
 
     def setattr(self, w_attr, w_value):
-        self.getcfield(w_attr).write(self._cdata, w_value)
-        keepalive_until_here(self)
+        cfield = self.getcfield(w_attr)
+        with self as ptr:
+            cfield.write(ptr, w_value)
 
     def call(self, args_w):
-        w_result = self.ctype.call(self._cdata, args_w)
-        keepalive_until_here(self)
+        with self as ptr:
+            w_result = self.ctype.call(ptr, args_w)
         return w_result
 
     def iter(self):
@@ -311,21 +335,21 @@
 
     @specialize.argtype(1)
     def write_raw_signed_data(self, source):
-        misc.write_raw_signed_data(self._cdata, source, self.ctype.size)
-        keepalive_until_here(self)
+        with self as ptr:
+            misc.write_raw_signed_data(ptr, source, self.ctype.size)
 
     @specialize.argtype(1)
     def write_raw_unsigned_data(self, source):
-        misc.write_raw_unsigned_data(self._cdata, source, self.ctype.size)
-        keepalive_until_here(self)
+        with self as ptr:
+            misc.write_raw_unsigned_data(ptr, source, self.ctype.size)
 
     def write_raw_float_data(self, source):
-        misc.write_raw_float_data(self._cdata, source, self.ctype.size)
-        keepalive_until_here(self)
+        with self as ptr:
+            misc.write_raw_float_data(ptr, source, self.ctype.size)
 
     def convert_to_object(self):
-        w_obj = self.ctype.convert_to_object(self._cdata)
-        keepalive_until_here(self)
+        with self as ptr:
+            w_obj = self.ctype.convert_to_object(ptr)
         return w_obj
 
     def get_array_length(self):
@@ -353,7 +377,7 @@
 
     @rgc.must_be_light_finalizer
     def __del__(self):
-        lltype.free(self._cdata, flavor='raw')
+        lltype.free(self._ptr, flavor='raw')
 
 
 class W_CDataNewOwning(W_CDataMem):
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -8,7 +8,6 @@
 from pypy.interpreter.typedef import TypeDef
 
 from rpython.rtyper.lltypesystem import rffi
-from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib.rarithmetic import ovfcheck
 
 from pypy.module._cffi_backend import cdataobj
@@ -49,8 +48,8 @@
             cdata = cdataobj.W_CDataNewOwning(space, datasize, self)
         #
         if not space.is_w(w_init, space.w_None):
-            self.convert_from_object(cdata._cdata, w_init)
-            keepalive_until_here(cdata)
+            with cdata as ptr:
+                self.convert_from_object(ptr, w_init)
         return cdata
 
     def _check_subscript_index(self, w_cdata, i):
@@ -119,8 +118,8 @@
         self.ctitem = ctitem
         self.cdata = cdata
         length = cdata.get_array_length()
-        self._next = cdata._cdata
-        self._stop = rffi.ptradd(cdata._cdata, length * ctitem.size)
+        self._next = cdata.unsafe_escaping_ptr()
+        self._stop = rffi.ptradd(self._next, length * ctitem.size)
 
     def iter_w(self):
         return self.space.wrap(self)
diff --git a/pypy/module/_cffi_backend/ctypeenum.py b/pypy/module/_cffi_backend/ctypeenum.py
--- a/pypy/module/_cffi_backend/ctypeenum.py
+++ b/pypy/module/_cffi_backend/ctypeenum.py
@@ -2,8 +2,6 @@
 Enums.
 """
 
-from rpython.rlib.objectmodel import keepalive_until_here
-
 from pypy.module._cffi_backend import misc
 from pypy.module._cffi_backend.ctypeprim import (W_CTypePrimitiveSigned,
     W_CTypePrimitiveUnsigned)
@@ -47,8 +45,8 @@
             return '%s: %s' % (value, s)
 
     def string(self, cdataobj, maxlen):
-        value = self._get_value(cdataobj._cdata)
-        keepalive_until_here(cdataobj)
+        with cdataobj as ptr:
+            value = self._get_value(ptr)
         try:
             s = self.enumvalues2erators[value]
         except KeyError:
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -177,8 +177,8 @@
         raise oefmt(space.w_AttributeError,
                     "cdata '%s' has no attribute '%s'", self.name, attr)
 
-    def copy_and_convert_to_object(self, cdata):
-        return self.convert_to_object(cdata)
+    def copy_and_convert_to_object(self, source):
+        return self.convert_to_object(source)
 
     # __________ app-level attributes __________
     def dir(self):
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -5,7 +5,6 @@
 import sys
 
 from rpython.rlib.rarithmetic import r_uint, r_ulonglong, intmask
-from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib import jit
 from rpython.rtyper.lltypesystem import lltype, rffi
 
@@ -53,7 +52,8 @@
         space = self.space
         if (isinstance(w_ob, cdataobj.W_CData) and
                isinstance(w_ob.ctype, ctypeptr.W_CTypePtrOrArray)):
-            value = rffi.cast(lltype.Signed, w_ob._cdata)
+            ptr = w_ob.unsafe_escaping_ptr()
+            value = rffi.cast(lltype.Signed, ptr)
             value = self._cast_result(value)
         elif space.isinstance_w(w_ob, space.w_str):
             value = self.cast_str(w_ob)
@@ -81,8 +81,8 @@
 
     def string(self, cdataobj, maxlen):
         if self.size == 1:
-            s = cdataobj._cdata[0]
-            keepalive_until_here(cdataobj)
+            with cdataobj as ptr:
+                s = ptr[0]
             return self.space.wrap(s)
         return W_CType.string(self, cdataobj, maxlen)
 
@@ -116,7 +116,8 @@
                 return s[0]
         if (isinstance(w_ob, cdataobj.W_CData) and
                isinstance(w_ob.ctype, W_CTypePrimitiveChar)):
-            return w_ob._cdata[0]
+            with w_ob as ptr:
+                return ptr[0]
         raise self._convert_error("string of length 1", w_ob)
 
     def convert_from_object(self, cdata, w_ob):
@@ -137,8 +138,8 @@
         return self.space.wrap(s)
 
     def string(self, cdataobj, maxlen):
-        w_res = self.convert_to_object(cdataobj._cdata)
-        keepalive_until_here(cdataobj)
+        with cdataobj as ptr:
+            w_res = self.convert_to_object(ptr)
         return w_res
 
     def _convert_to_unichar(self, w_ob):
@@ -149,7 +150,8 @@
                 return s[0]
         if (isinstance(w_ob, cdataobj.W_CData) and
                isinstance(w_ob.ctype, W_CTypePrimitiveUniChar)):
-            return rffi.cast(rffi.CWCHARP, w_ob._cdata)[0]
+            with w_ob as ptr:
+                return rffi.cast(rffi.CWCHARP, ptr)[0]
         raise self._convert_error("unicode string of length 1", w_ob)
 
     def convert_from_object(self, cdata, w_ob):
@@ -219,13 +221,15 @@
         if self.size == rffi.sizeof(rffi.LONG):
             from rpython.rlib.rrawarray import populate_list_from_raw_array
             res = []
-            buf = rffi.cast(rffi.LONGP, w_cdata._cdata)
             length = w_cdata.get_array_length()
-            populate_list_from_raw_array(res, buf, length)
+            with w_cdata as ptr:
+                buf = rffi.cast(rffi.LONGP, ptr)
+                populate_list_from_raw_array(res, buf, length)
             return res
         elif self.value_smaller_than_long:
             res = [0] * w_cdata.get_array_length()
-            misc.unpack_list_from_raw_array(res, w_cdata._cdata, self.size)
+            with w_cdata as ptr:
+                misc.unpack_list_from_raw_array(res, ptr, self.size)
             return res
         return None
 
@@ -308,8 +312,8 @@
     def unpack_list_of_int_items(self, w_cdata):
         if self.value_fits_long:
             res = [0] * w_cdata.get_array_length()
-            misc.unpack_unsigned_list_from_raw_array(res, w_cdata._cdata,
-                                                     self.size)
+            with w_cdata as ptr:
+                misc.unpack_unsigned_list_from_raw_array(res, ptr, self.size)
             return res
         return None
 
@@ -363,8 +367,8 @@
         if not isinstance(self, W_CTypePrimitiveLongDouble):
             w_cdata.write_raw_float_data(value)
         else:
-            self._to_longdouble_and_write(value, w_cdata._cdata)
-            keepalive_until_here(w_cdata)
+            with w_cdata as ptr:
+                self._to_longdouble_and_write(value, ptr)
         return w_cdata
 
     def cast_to_int(self, cdata):
@@ -387,13 +391,15 @@
         if self.size == rffi.sizeof(rffi.DOUBLE):
             from rpython.rlib.rrawarray import populate_list_from_raw_array
             res = []
-            buf = rffi.cast(rffi.DOUBLEP, w_cdata._cdata)
             length = w_cdata.get_array_length()
-            populate_list_from_raw_array(res, buf, length)
+            with w_cdata as ptr:
+                buf = rffi.cast(rffi.DOUBLEP, ptr)
+                populate_list_from_raw_array(res, buf, length)
             return res
         elif self.size == rffi.sizeof(rffi.FLOAT):
             res = [0.0] * w_cdata.get_array_length()
-            misc.unpack_cfloat_list_from_raw_array(res, w_cdata._cdata)
+            with w_cdata as ptr:
+                misc.unpack_cfloat_list_from_raw_array(res, ptr)
             return res
         return None
 
@@ -423,8 +429,8 @@
     def cast(self, w_ob):
         if (isinstance(w_ob, cdataobj.W_CData) and
                 isinstance(w_ob.ctype, W_CTypePrimitiveLongDouble)):
-            w_cdata = self.convert_to_object(w_ob._cdata)
-            keepalive_until_here(w_ob)
+            with w_ob as ptr:
+                w_cdata = self.convert_to_object(ptr)
             return w_cdata
         else:
             return W_CTypePrimitiveFloat.cast(self, w_ob)
@@ -451,16 +457,16 @@
 
     def convert_to_object(self, cdata):
         w_cdata = cdataobj.W_CDataMem(self.space, self.size, self)
-        self._copy_longdouble(cdata, w_cdata._cdata)
-        keepalive_until_here(w_cdata)
+        with w_cdata as ptr:
+            self._copy_longdouble(cdata, ptr)
         return w_cdata
 
     def convert_from_object(self, cdata, w_ob):
         space = self.space
         if (isinstance(w_ob, cdataobj.W_CData) and
                 isinstance(w_ob.ctype, W_CTypePrimitiveLongDouble)):
-            self._copy_longdouble(w_ob._cdata, cdata)
-            keepalive_until_here(w_ob)
+            with w_ob as ptr:
+                self._copy_longdouble(ptr, cdata)
         else:
             value = space.float_w(space.float(w_ob))
             self._to_longdouble_and_write(value, cdata)
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -3,7 +3,6 @@
 """
 
 from rpython.rlib import rposix
-from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rtyper.annlowlevel import llstr, llunicode
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -49,7 +48,7 @@
         space = self.space
         if (isinstance(w_ob, cdataobj.W_CData) and
                 isinstance(w_ob.ctype, W_CTypePtrOrArray)):
-            value = w_ob._cdata
+            value = w_ob.unsafe_escaping_ptr()
         else:
             value = misc.as_unsigned_long(space, w_ob, strict=False)
             value = rffi.cast(rffi.CCHARP, value)
@@ -108,34 +107,33 @@
     def string(self, cdataobj, maxlen):
         space = self.space
         if isinstance(self.ctitem, ctypeprim.W_CTypePrimitive):
-            cdata = cdataobj._cdata
-            if not cdata:
-                raise oefmt(space.w_RuntimeError, "cannot use string() on %s",
-                            space.str_w(cdataobj.repr()))
-            #
-            from pypy.module._cffi_backend import ctypearray
-            length = maxlen
-            if length < 0 and isinstance(self, ctypearray.W_CTypeArray):
-                length = cdataobj.get_array_length()
-            #
-            # pointer to a primitive type of size 1: builds and returns a str
-            if self.ctitem.size == rffi.sizeof(lltype.Char):
-                if length < 0:
-                    s = rffi.charp2str(cdata)
-                else:
-                    s = rffi.charp2strn(cdata, length)
-                keepalive_until_here(cdataobj)
-                return space.wrap(s)
-            #
-            # pointer to a wchar_t: builds and returns a unicode
-            if self.is_unichar_ptr_or_array():
-                cdata = rffi.cast(rffi.CWCHARP, cdata)
-                if length < 0:
-                    u = rffi.wcharp2unicode(cdata)
-                else:
-                    u = rffi.wcharp2unicoden(cdata, length)
-                keepalive_until_here(cdataobj)
-                return space.wrap(u)
+            with cdataobj as ptr:
+                if not ptr:
+                    raise oefmt(space.w_RuntimeError,
+                                "cannot use string() on %s",
+                                space.str_w(cdataobj.repr()))
+                #
+                from pypy.module._cffi_backend import ctypearray
+                length = maxlen
+                if length < 0 and isinstance(self, ctypearray.W_CTypeArray):
+                    length = cdataobj.get_array_length()
+                #
+                # pointer to a primitive type of size 1: builds and returns a str
+                if self.ctitem.size == rffi.sizeof(lltype.Char):
+                    if length < 0:
+                        s = rffi.charp2str(ptr)
+                    else:
+                        s = rffi.charp2strn(ptr, length)
+                    return space.wrap(s)
+                #
+                # pointer to a wchar_t: builds and returns a unicode
+                if self.is_unichar_ptr_or_array():
+                    cdata = rffi.cast(rffi.CWCHARP, ptr)
+                    if length < 0:
+                        u = rffi.wcharp2unicode(cdata)
+                    else:
+                        u = rffi.wcharp2unicoden(cdata, length)
+                    return space.wrap(u)
         #
         return W_CType.string(self, cdataobj, maxlen)
 
@@ -162,7 +160,7 @@
             if not (self.can_cast_anything or other.can_cast_anything):
                 raise self._convert_error("compatible pointer", w_ob)
 
-        rffi.cast(rffi.CCHARPP, cdata)[0] = w_ob._cdata
+        rffi.cast(rffi.CCHARPP, cdata)[0] = w_ob.unsafe_escaping_ptr()
 
     def _alignof(self):
         from pypy.module._cffi_backend import newtype
@@ -206,8 +204,8 @@
                     lltype.nullptr(rffi.CCHARP.TO), w_init, datasize)
             #
             cdatastruct = cdataobj.W_CDataNewOwning(space, datasize, ctitem)
-            cdata = cdataobj.W_CDataPtrToStructOrUnion(space,
-                                                       cdatastruct._cdata,
+            ptr = cdatastruct.unsafe_escaping_ptr()
+            cdata = cdataobj.W_CDataPtrToStructOrUnion(space, ptr,
                                                        self, cdatastruct)
         else:
             if self.is_char_or_unichar_ptr_or_array():
@@ -215,8 +213,8 @@
             cdata = cdataobj.W_CDataNewOwning(space, datasize, self)
         #
         if not space.is_w(w_init, space.w_None):
-            ctitem.convert_from_object(cdata._cdata, w_init)
-            keepalive_until_here(cdata)
+            with cdata as ptr:
+                ctitem.convert_from_object(ptr, w_init)
         return cdata
 
     def _check_subscript_index(self, w_cdata, i):
@@ -332,8 +330,9 @@
         ctype2 = cdata.ctype
         if (isinstance(ctype2, W_CTypeStructOrUnion) or
                 isinstance(ctype2, W_CTypePtrOrArray)):
-            ptrdata = rffi.ptradd(cdata._cdata, offset)
-            return cdataobj.W_CData(space, ptrdata, self)
+            ptr = cdata.unsafe_escaping_ptr()
+            ptr = rffi.ptradd(ptr, offset)
+            return cdataobj.W_CData(space, ptr, self)
         else:
             raise OperationError(space.w_TypeError,
                     space.wrap("expected a cdata struct/union/array/pointer"
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -7,7 +7,6 @@
 from pypy.interpreter.typedef import TypeDef, interp_attrproperty
 
 from rpython.rlib import jit
-from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib.rarithmetic import r_uint, r_ulonglong, r_longlong, intmask
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -57,12 +56,12 @@
         self.check_complete()
         return cdataobj.W_CData(space, cdata, self)
 
-    def copy_and_convert_to_object(self, cdata):
+    def copy_and_convert_to_object(self, source):
         space = self.space
         self.check_complete()
         ob = cdataobj.W_CDataNewOwning(space, self.size, self)
-        misc._raw_memcopy(cdata, ob._cdata, self.size)
-        keepalive_until_here(ob)
+        with ob as target:
+            misc._raw_memcopy(source, target, self.size)
         return ob
 
     def typeoffsetof_field(self, fieldname, following):
@@ -80,8 +79,8 @@
     def _copy_from_same(self, cdata, w_ob):
         if isinstance(w_ob, cdataobj.W_CData):
             if w_ob.ctype is self and self.size >= 0:
-                misc._raw_memcopy(w_ob._cdata, cdata, self.size)
-                keepalive_until_here(w_ob)
+                with w_ob as ptr:
+                    misc._raw_memcopy(ptr, cdata, self.size)
                 return True
         return False
 
diff --git a/pypy/module/_cffi_backend/ctypevoid.py b/pypy/module/_cffi_backend/ctypevoid.py
--- a/pypy/module/_cffi_backend/ctypevoid.py
+++ b/pypy/module/_cffi_backend/ctypevoid.py
@@ -13,5 +13,5 @@
     def __init__(self, space):
         W_CType.__init__(self, space, -1, "void", len("void"))
 
-    def copy_and_convert_to_object(self, cdata):
+    def copy_and_convert_to_object(self, source):
         return self.space.w_None
diff --git a/pypy/module/_cffi_backend/handle.py b/pypy/module/_cffi_backend/handle.py
--- a/pypy/module/_cffi_backend/handle.py
+++ b/pypy/module/_cffi_backend/handle.py
@@ -34,8 +34,9 @@
         raise oefmt(space.w_TypeError,
                     "expected a 'cdata' object with a 'void *' out of "
                     "new_handle(), got '%s'", ctype.name)
-    index = rffi.cast(lltype.Signed, w_cdata._cdata)
-    original_cdataobj = get(space).fetch_handle(index - 1)
+    with w_cdata as ptr:
+        index = rffi.cast(lltype.Signed, ptr)
+        original_cdataobj = get(space).fetch_handle(index - 1)
     #
     if isinstance(original_cdataobj, cdataobj.W_CDataHandle):
         return original_cdataobj.w_keepalive
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -3,7 +3,7 @@
 from pypy.interpreter.error import OperationError
 
 from rpython.rlib import jit
-from rpython.rlib.objectmodel import keepalive_until_here, specialize
+from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rarithmetic import r_uint, r_ulonglong
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
@@ -272,11 +272,11 @@
     from pypy.module._cffi_backend.ctypeprim import W_CTypePrimitiveLongDouble
     is_cdata = isinstance(w_ob, W_CData)
     if is_cdata and isinstance(w_ob.ctype, W_CTypePrimitiveFloat):
-        if isinstance(w_ob.ctype, W_CTypePrimitiveLongDouble):
-            result = is_nonnull_longdouble(w_ob._cdata)
-        else:
-            result = is_nonnull_float(w_ob._cdata, w_ob.ctype.size)
-        keepalive_until_here(w_ob)
+        with w_ob as ptr:
+            if isinstance(w_ob.ctype, W_CTypePrimitiveLongDouble):
+                result = is_nonnull_longdouble(ptr)
+            else:
+                result = is_nonnull_float(ptr, w_ob.ctype.size)
         return result
     #
     if not is_cdata and space.lookup(w_ob, '__float__') is not None:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3249,4 +3249,4 @@
 
 def test_version():
     # this test is here mostly for PyPy
-    assert __version__ == "0.8.6+"
+    assert __version__ == "0.9.0"
diff --git a/pypy/module/_random/test/test_random.py b/pypy/module/_random/test/test_random.py
--- a/pypy/module/_random/test/test_random.py
+++ b/pypy/module/_random/test/test_random.py
@@ -102,3 +102,10 @@
                 self.x = x
         r = R(x=15)
         assert r.x == 15
+
+    def test_exact_result(self):
+        # this passes on CPython 2.7.9 on Linux 32 and Linux 64
+        import _random
+        rnd = _random.Random(-3**31)
+        x = rnd.random()
+        assert x == 0.8181851342382107
diff --git a/pypy/module/_rawffi/array.py b/pypy/module/_rawffi/array.py
--- a/pypy/module/_rawffi/array.py
+++ b/pypy/module/_rawffi/array.py
@@ -15,7 +15,7 @@
 from pypy.module._rawffi.interp_rawffi import unpack_shape_with_length
 from pypy.module._rawffi.interp_rawffi import read_ptr, write_ptr
 from rpython.rlib.rarithmetic import r_uint
-from rpython.rlib import rgc
+from rpython.rlib import rgc, clibffi
 
 
 class W_Array(W_DataShape):
@@ -84,14 +84,11 @@
 
 class W_ArrayInstance(W_DataInstance):
     def __init__(self, space, shape, length, address=r_uint(0)):
-        # Workaround for a strange behavior of libffi: make sure that
-        # we always have at least 8 bytes.  For W_ArrayInstances that are
-        # used as the result value of a function call, ffi_call() writes
-        # 8 bytes into it even if the function's result type asks for less.
-        # This strange behavior is documented.
         memsize = shape.size * length
-        if memsize < 8:
-            memsize = 8
+        # For W_ArrayInstances that are used as the result value of a
+        # function call, ffi_call() writes 8 bytes into it even if the
+        # function's result type asks for less.
+        memsize = clibffi.adjust_return_size(memsize)
         W_DataInstance.__init__(self, space, memsize, address)
         self.length = length
         self.shape = shape
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -495,6 +495,7 @@
         try:
             if self.resshape is not None:
                 result = self.resshape.allocate(space, 1, autofree=True)
+                # adjust_return_size() was used here on result.ll_buffer
                 self.ptr.call(args_ll, result.ll_buffer)
                 return space.wrap(result)
             else:
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -1,4 +1,4 @@
-from rpython.rlib import jit
+from rpython.rlib import jit, rgc
 from rpython.rlib.buffer import Buffer
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib.rarithmetic import ovfcheck, widen
@@ -698,11 +698,9 @@
                                          self.space.wrap(msg))
             return result
 
+        @rgc.must_be_light_finalizer
         def __del__(self):
-            # note that we don't call clear_all_weakrefs here because
-            # an array with freed buffer is ok to see - it's just empty with 0
-            # length
-            self.setlen(0)
+            lltype.free(self.buffer, flavor='raw')
 
         def setlen(self, size, zero=False, overallocate=True):
             if size > 0:
diff --git a/pypy/module/cppyy/capi/loadable_capi.py b/pypy/module/cppyy/capi/loadable_capi.py
--- a/pypy/module/cppyy/capi/loadable_capi.py
+++ b/pypy/module/cppyy/capi/loadable_capi.py
@@ -259,7 +259,8 @@
         if not objectmodel.we_are_translated():
             leakfinder.remember_free(c_call.ctype.cif_descr._obj0)
         state.capi_calls[name] = c_call
-    return c_call.ctype.rcall(c_call._cdata, args)
+    with c_call as ptr:
+        return c_call.ctype.rcall(ptr, args)
 
 def _cdata_to_cobject(space, w_cdata):
     return rffi.cast(C_OBJECT, space.uint_w(w_cdata))
@@ -271,8 +272,9 @@
     return rffi.cast(rffi.LONG, space.int_w(w_cdata))
 
 def _cdata_to_ptr(space, w_cdata): # TODO: this is both a hack and dreadfully slow
-    return rffi.cast(rffi.VOIDP,
-        space.interp_w(cdataobj.W_CData, w_cdata, can_be_None=False)._cdata)
+    w_cdata = space.interp_w(cdataobj.W_CData, w_cdata, can_be_None=False)
+    ptr = w_cdata.unsafe_escaping_ptr()
+    return rffi.cast(rffi.VOIDP, ptr)
 
 def c_load_dictionary(name):
     return libffi.CDLL(name)
diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py
--- a/pypy/module/micronumpy/base.py
+++ b/pypy/module/micronumpy/base.py
@@ -46,7 +46,7 @@
     @staticmethod
     def from_shape_and_storage(space, shape, storage, dtype, storage_bytes=-1,
                                order='C', owning=False, w_subtype=None,
-                               w_base=None, writable=True, strides=None):
+                               w_base=None, writable=True, strides=None, start=0):
         from pypy.module.micronumpy import concrete
         from pypy.module.micronumpy.strides import (calc_strides,
                                                     calc_backstrides)
@@ -75,8 +75,9 @@
                 raise OperationError(space.w_ValueError,
                         space.wrap("Cannot have owning=True when specifying a buffer"))
             if writable:
-                impl = concrete.ConcreteArrayWithBase(shape, dtype, order, strides,
-                                                      backstrides, storage, w_base)
+                impl = concrete.ConcreteArrayWithBase(shape, dtype, order,
+                                    strides, backstrides, storage, w_base,
+                                    start=start)
             else:
                 impl = concrete.ConcreteNonWritableArrayWithBase(shape, dtype, order,
                                                                  strides, backstrides,
@@ -128,6 +129,9 @@
     def get_order(self):
         return self.implementation.order
 
+    def get_start(self):
+        return self.implementation.start
+
     def ndims(self):
         return len(self.get_shape())
     ndims._always_inline_ = True
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -12,6 +12,7 @@
 from pypy.module.micronumpy.strides import (Chunk, Chunks, NewAxisChunk,
     RecordChunk, calc_strides, calc_new_strides, shape_agreement,
     calculate_broadcast_strides, calc_backstrides)
+from rpython.rlib.objectmodel import keepalive_until_here
 
 
 class BaseConcreteArray(object):
@@ -312,12 +313,15 @@
         l_w = [w_res.descr_getitem(space, space.wrap(d)) for d in range(nd)]
         return space.newtuple(l_w)
 
-    def get_storage_as_int(self, space):
-        return rffi.cast(lltype.Signed, self.storage) + self.start
-
-    def get_storage(self):
+    ##def get_storage(self):
+    ##    return self.storage
+    ## use a safer context manager
+    def __enter__(self):
         return self.storage
 
+    def __exit__(self, typ, value, traceback):
+        keepalive_until_here(self)
+        
     def get_buffer(self, space, readonly):
         return ArrayBuffer(self, readonly)
 
@@ -331,7 +335,7 @@
 
 
 class ConcreteArrayNotOwning(BaseConcreteArray):
-    def __init__(self, shape, dtype, order, strides, backstrides, storage):
+    def __init__(self, shape, dtype, order, strides, backstrides, storage, start=0):
         make_sure_not_resized(shape)
         make_sure_not_resized(strides)
         make_sure_not_resized(backstrides)
@@ -342,6 +346,7 @@
         self.strides = strides
         self.backstrides = backstrides
         self.storage = storage
+        self.start = start
 
     def fill(self, space, box):
         self.dtype.itemtype.fill(self.storage, self.dtype.elsize,
@@ -350,7 +355,7 @@
     def set_shape(self, space, orig_array, new_shape):
         strides, backstrides = calc_strides(new_shape, self.dtype,
                                                     self.order)
-        return SliceArray(0, strides, backstrides, new_shape, self,
+        return SliceArray(self.start, strides, backstrides, new_shape, self,
                           orig_array)
 
     def set_dtype(self, space, dtype):
@@ -384,9 +389,10 @@
 
 
 class ConcreteArrayWithBase(ConcreteArrayNotOwning):
-    def __init__(self, shape, dtype, order, strides, backstrides, storage, orig_base):
+    def __init__(self, shape, dtype, order, strides, backstrides, storage,
+                 orig_base, start=0):
         ConcreteArrayNotOwning.__init__(self, shape, dtype, order,
-                                        strides, backstrides, storage)
+                                        strides, backstrides, storage, start)
         self.orig_base = orig_base
 
     def base(self):
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -99,10 +99,12 @@
             for i in range(w_object.get_size()):
                 elems_w[i] = w_object.implementation.getitem(i * elsize)
         else:
-            sz = support.product(w_object.get_shape()) * dtype.elsize
-            return W_NDimArray.from_shape_and_storage(space,
-                w_object.get_shape(),w_object.implementation.storage,
-                dtype, storage_bytes=sz, w_base=w_object)
+            imp = w_object.implementation
+            with imp as storage:
+                sz = support.product(w_object.get_shape()) * dtype.elsize
+                return W_NDimArray.from_shape_and_storage(space,
+                    w_object.get_shape(), storage, dtype, storage_bytes=sz, 
+                    w_base=w_object, start=imp.start)
     else:
         # not an array
         shape, elems_w = strides.find_shape_and_elems(space, w_object, dtype)
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -604,14 +604,15 @@
     iter, state = arr.create_iter()
     w_res_str = W_NDimArray.from_shape(space, [1], arr.get_dtype(), order='C')
     itemsize = arr.get_dtype().elsize
-    res_str_casted = rffi.cast(rffi.CArrayPtr(lltype.Char),
-                               w_res_str.implementation.get_storage_as_int(space))
-    while not iter.done(state):
-        w_res_str.implementation.setitem(0, iter.getitem(state))
-        for i in range(itemsize):
-            builder.append(res_str_casted[i])
-        state = iter.next(state)
-    return builder.build()
+    with w_res_str.implementation as storage:
+        res_str_casted = rffi.cast(rffi.CArrayPtr(lltype.Char),
+                               support.get_storage_as_int(storage))
+        while not iter.done(state):
+            w_res_str.implementation.setitem(0, iter.getitem(state))
+            for i in range(itemsize):
+                builder.append(res_str_casted[i])
+            state = iter.next(state)
+        return builder.build()
 
 getitem_int_driver = jit.JitDriver(name = 'numpy_getitem_int',
                                    greens = ['shapelen', 'indexlen',
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -532,20 +532,25 @@
             self.get_dtype(), storage_bytes=sz, w_base=self)
 
     def descr_array_iface(self, space):
-        addr = self.implementation.get_storage_as_int(space)
-        # will explode if it can't
-        w_d = space.newdict()
-        space.setitem_str(w_d, 'data',
-                          space.newtuple([space.wrap(addr), space.w_False]))
-        space.setitem_str(w_d, 'shape', self.descr_get_shape(space))
-        space.setitem_str(w_d, 'typestr', self.get_dtype().descr_get_str(space))
-        if self.implementation.order == 'C':
-            # Array is contiguous, no strides in the interface.
-            strides = space.w_None
-        else:
-            strides = self.descr_get_strides(space)
-        space.setitem_str(w_d, 'strides', strides)
-        return w_d
+        '''
+        Note: arr.__array__.data[0] is a pointer so arr must be kept alive
+              while it is in use
+        '''
+        with self.implementation as storage:
+            addr = support.get_storage_as_int(storage, self.get_start())
+            # will explode if it can't
+            w_d = space.newdict()
+            space.setitem_str(w_d, 'data',
+                              space.newtuple([space.wrap(addr), space.w_False]))
+            space.setitem_str(w_d, 'shape', self.descr_get_shape(space))
+            space.setitem_str(w_d, 'typestr', self.get_dtype().descr_get_str(space))
+            if self.implementation.order == 'C':
+                # Array is contiguous, no strides in the interface.
+                strides = space.w_None
+            else:
+                strides = self.descr_get_strides(space)
+            space.setitem_str(w_d, 'strides', strides)
+            return w_d
 
     w_pypy_data = None
 
@@ -1165,7 +1170,8 @@
                 builder.append(box.raw_str())
                 state = iter.next(state)
         else:
-            builder.append_charpsize(self.implementation.get_storage(),
+            with self.implementation as storage:
+                builder.append_charpsize(storage,
                                      self.implementation.get_storage_size())
 
         state = space.newtuple([
diff --git a/pypy/module/micronumpy/selection.py b/pypy/module/micronumpy/selection.py
--- a/pypy/module/micronumpy/selection.py
+++ b/pypy/module/micronumpy/selection.py
@@ -33,14 +33,14 @@
             self.values = values
             self.indexes = indexes
 
-        def getitem(self, item):
+        def getitem(self, idx):
             if count < 2:
-                v = raw_storage_getitem(TP, self.values, item * self.stride_size
+                v = raw_storage_getitem(TP, self.values, idx * self.stride_size
                                     + self.start)
             else:
                 v = []
                 for i in range(count):
-                    _v = raw_storage_getitem(TP, self.values, item * self.stride_size
+                    _v = raw_storage_getitem(TP, self.values, idx * self.stride_size
                                     + self.start + step * i)
                     v.append(_v)
             if comp_type == 'int':
@@ -52,7 +52,7 @@
             else:
                 raise NotImplementedError('cannot reach')
             return (v, raw_storage_getitem(lltype.Signed, self.indexes,
-                                           item * self.index_stride_size +
+                                           idx * self.index_stride_size +
                                            self.index_start))
 
         def setitem(self, idx, item):
@@ -134,37 +134,37 @@
         # create array of indexes
         dtype = descriptor.get_dtype_cache(space).w_longdtype
         index_arr = W_NDimArray.from_shape(space, arr.get_shape(), dtype)
-        storage = index_arr.implementation.get_storage()
-        if len(arr.get_shape()) == 1:
-            for i in range(arr.get_size()):
-                raw_storage_setitem(storage, i * INT_SIZE, i)
-            r = Repr(INT_SIZE, itemsize, arr.get_size(), arr.get_storage(),
-                     storage, 0, arr.start)
-            ArgSort(r).sort()
-        else:
-            shape = arr.get_shape()
-            if axis < 0:
-                axis = len(shape) + axis
-            if axis < 0 or axis >= len(shape):
-                raise oefmt(space.w_IndexError, "Wrong axis %d", axis)
-            arr_iter = AllButAxisIter(arr, axis)
-            arr_state = arr_iter.reset()
-            index_impl = index_arr.implementation
-            index_iter = AllButAxisIter(index_impl, axis)
-            index_state = index_iter.reset()
-            stride_size = arr.strides[axis]
-            index_stride_size = index_impl.strides[axis]
-            axis_size = arr.shape[axis]
-            while not arr_iter.done(arr_state):
-                for i in range(axis_size):
-                    raw_storage_setitem(storage, i * index_stride_size +
-                                        index_state.offset, i)
-                r = Repr(index_stride_size, stride_size, axis_size,
-                         arr.get_storage(), storage, index_state.offset, arr_state.offset)
+        with index_arr.implementation as storage, arr as arr_storage:
+            if len(arr.get_shape()) == 1:
+                for i in range(arr.get_size()):
+                    raw_storage_setitem(storage, i * INT_SIZE, i)
+                r = Repr(INT_SIZE, itemsize, arr.get_size(), arr_storage,
+                         storage, 0, arr.start)
                 ArgSort(r).sort()
-                arr_state = arr_iter.next(arr_state)
-                index_state = index_iter.next(index_state)
-        return index_arr
+            else:
+                shape = arr.get_shape()
+                if axis < 0:
+                    axis = len(shape) + axis
+                if axis < 0 or axis >= len(shape):
+                    raise oefmt(space.w_IndexError, "Wrong axis %d", axis)
+                arr_iter = AllButAxisIter(arr, axis)
+                arr_state = arr_iter.reset()
+                index_impl = index_arr.implementation
+                index_iter = AllButAxisIter(index_impl, axis)
+                index_state = index_iter.reset()
+                stride_size = arr.strides[axis]
+                index_stride_size = index_impl.strides[axis]
+                axis_size = arr.shape[axis]
+                while not arr_iter.done(arr_state):
+                    for i in range(axis_size):
+                        raw_storage_setitem(storage, i * index_stride_size +
+                                            index_state.offset, i)
+                    r = Repr(index_stride_size, stride_size, axis_size,
+                         arr_storage, storage, index_state.offset, arr_state.offset)
+                    ArgSort(r).sort()
+                    arr_state = arr_iter.next(arr_state)
+                    index_state = index_iter.next(index_state)
+            return index_arr
 
     return argsort
 
@@ -282,25 +282,25 @@
             axis = -1
         else:
             axis = space.int_w(w_axis)
-        # create array of indexes
-        if len(arr.get_shape()) == 1:
-            r = Repr(itemsize, arr.get_size(), arr.get_storage(),
-                     arr.start)
-            ArgSort(r).sort()
-        else:
-            shape = arr.get_shape()
-            if axis < 0:
-                axis = len(shape) + axis
-            if axis < 0 or axis >= len(shape):
-                raise oefmt(space.w_IndexError, "Wrong axis %d", axis)
-            arr_iter = AllButAxisIter(arr, axis)
-            arr_state = arr_iter.reset()
-            stride_size = arr.strides[axis]
-            axis_size = arr.shape[axis]
-            while not arr_iter.done(arr_state):
-                r = Repr(stride_size, axis_size, arr.get_storage(), arr_state.offset)
+        with arr as storage:
+            if len(arr.get_shape()) == 1:
+                r = Repr(itemsize, arr.get_size(), storage,
+                         arr.start)
                 ArgSort(r).sort()
-                arr_state = arr_iter.next(arr_state)
+            else:
+                shape = arr.get_shape()
+                if axis < 0:
+                    axis = len(shape) + axis
+                if axis < 0 or axis >= len(shape):
+                    raise oefmt(space.w_IndexError, "Wrong axis %d", axis)
+                arr_iter = AllButAxisIter(arr, axis)
+                arr_state = arr_iter.reset()
+                stride_size = arr.strides[axis]
+                axis_size = arr.shape[axis]
+                while not arr_iter.done(arr_state):
+                    r = Repr(stride_size, axis_size, storage, arr_state.offset)
+                    ArgSort(r).sort()
+                    arr_state = arr_iter.next(arr_state)
 
     return sort
 
diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py
--- a/pypy/module/micronumpy/support.py
+++ b/pypy/module/micronumpy/support.py
@@ -1,6 +1,7 @@
 from pypy.interpreter.error import OperationError, oefmt
 from rpython.rlib import jit
 from rpython.rlib.rarithmetic import ovfcheck
+from rpython.rtyper.lltypesystem import rffi, lltype
 
 
 def issequence_w(space, w_obj):
@@ -147,3 +148,7 @@
     if cur_core_dim == 0:
         ufunc.core_enabled = False
     return 0 # for historical reasons, any failures will raise
+
+def get_storage_as_int(storage, start=0):
+        return rffi.cast(lltype.Signed, storage) + start
+
diff --git a/pypy/module/micronumpy/test/test_subtype.py b/pypy/module/micronumpy/test/test_subtype.py
--- a/pypy/module/micronumpy/test/test_subtype.py
+++ b/pypy/module/micronumpy/test/test_subtype.py
@@ -2,7 +2,7 @@
 
 
 class AppTestSupport(BaseNumpyAppTest):
-    spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii"])
+    spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii", "mmap"])
 
     def setup_class(cls):
         BaseNumpyAppTest.setup_class.im_func(cls)
@@ -476,3 +476,120 @@
         a = self.SubType(array([[1, 2], [3, 4]]))
         b = array(a, subok=False)
         assert type(b) is ndarray
+    
+    def test_numpypy_mmap(self):
+        # issue #21 on pypy/numpy 
+        from numpy import array, ndarray, arange, dtype as dtypedescr
+        import mmap
+        import os.path
+        from tempfile import mkdtemp
+        import os.path as path
+        valid_filemodes = ["r", "c", "r+", "w+"]
+        writeable_filemodes = ["r+", "w+"]
+        mode_equivalents = {
+            "readonly":"r",
+            "copyonwrite":"c",
+            "readwrite":"r+",
+            "write":"w+"
+            }
+
+        class memmap(ndarray):
+            def __new__(subtype, filename, dtype='uint8', mode='r+', offset=0, shape=None, order='C'):
+                # Import here to minimize 'import numpy' overhead
+                try:
+                    mode = mode_equivalents[mode]
+                except KeyError:
+                    if mode not in valid_filemodes:
+                        raise ValueError("mode must be one of %s" %
+                                         (valid_filemodes + list(mode_equivalents.keys())))
+
+                if hasattr(filename, 'read'):
+                    fid = filename
+                    own_file = False
+                else:
+                    fid = open(filename, (mode == 'c' and 'r' or mode)+'b')
+                    own_file = True
+
+                if (mode == 'w+') and shape is None:
+                    raise ValueError("shape must be given")
+
+                fid.seek(0, 2)
+                flen = fid.tell()
+                descr = dtypedescr(dtype)
+                _dbytes = descr.itemsize
+
+                if shape is None:
+                    bytes = flen - offset
+                    if (bytes % _dbytes):
+                        fid.close()
+                        raise ValueError("Size of available data is not a "
+                                "multiple of the data-type size.")
+                    size = bytes // _dbytes
+                    shape = (size,)
+                else:
+                    if not isinstance(shape, tuple):
+                        shape = (shape,)
+                    size = 1
+                    for k in shape:
+                        size *= k
+
+                bytes = long(offset + size*_dbytes)
+
+                if mode == 'w+' or (mode == 'r+' and flen < bytes):
+                    fid.seek(bytes - 1, 0)
+                    fid.write('\0')
+                    fid.flush()
+
+                if mode == 'c':
+                    acc = mmap.ACCESS_COPY
+                elif mode == 'r':
+                    acc = mmap.ACCESS_READ
+                else:
+                    acc = mmap.ACCESS_WRITE
+
+                start = offset - offset % mmap.ALLOCATIONGRANULARITY
+                bytes -= start
+                offset -= start
+                mm = mmap.mmap(fid.fileno(), bytes, access=acc, offset=start)
+
+                self = ndarray.__new__(subtype, shape, dtype=descr, buffer=mm,
+                    offset=offset, order=order)
+                self._mmap = mm
+                self.offset = offset
+                self.mode = mode
+
+                if isinstance(filename, basestring):
+                    self.filename = os.path.abspath(filename)
+                # py3 returns int for TemporaryFile().name
+                elif (hasattr(filename, "name") and
+                      isinstance(filename.name, basestring)):
+                    self.filename = os.path.abspath(filename.name)
+                # same as memmap copies (e.g. memmap + 1)
+                else:
+                    self.filename = None
+
+                if own_file:
+                    fid.close()
+
+                return self
+
+            def flush(self):
+                if self.base is not None and hasattr(self.base, 'flush'):
+                    self.base.flush() 
+
+        def asarray(obj, itemsize=None, order=None):
+            return array(obj, itemsize, copy=False, order=order)
+
+        filename = path.join(mkdtemp(), 'newfile.dat')
+        data = arange(10*10*36).reshape(10, 10, 36)
+        fp = memmap(filename, dtype='float32', mode='w+', shape=data.shape)
+        vals = [   242,    507,    255,    505,    315,    316,    308,    506,
+          309,    255,    211,    505,    315,    316,    308,    506,
+          309,    255,    255,    711,    194,    232,    711,    711,
+          709,    710,    709,    710,    882,    897,    711,    245,
+          711,    711,    168,    245]
+        fp[:] = data
+        fp[5:6][:,4] = vals
+        a = asarray(fp[5:6][:,4])
+        assert (a == vals).all()
+
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -238,15 +238,15 @@
             'getfield_gc': 5,
             'getfield_gc_pure': 51,
             'guard_class': 3,
-            'guard_false': 13,
+            'guard_false': 12,
             'guard_nonnull': 11,
             'guard_nonnull_class': 3,
             'guard_not_invalidated': 2,
             'guard_true': 10,
-            'guard_value': 5,
+            'guard_value': 6,
             'int_add': 13,
             'int_ge': 4,
-            'int_is_true': 4,
+            'int_is_true': 3,
             'int_is_zero': 4,
             'int_le': 2,
             'int_lt': 3,
@@ -616,15 +616,15 @@
             'getfield_gc': 6,
             'getfield_gc_pure': 63,
             'guard_class': 5,
-            'guard_false': 20,
+            'guard_false': 19,
             'guard_nonnull': 6,
             'guard_nonnull_class': 1,
             'guard_not_invalidated': 3,
             'guard_true': 16,
-            'guard_value': 2,
+            'guard_value': 3,
             'int_add': 24,
             'int_ge': 4,
-            'int_is_true': 6,
+            'int_is_true': 5,
             'int_is_zero': 4,
             'int_le': 5,
             'int_lt': 7,
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -180,13 +180,16 @@
         raw_storage_setitem_unaligned(storage, i + offset, value)
 
     def read(self, arr, i, offset, dtype=None):
-        return self.box(self._read(arr.storage, i, offset))
+        with arr as storage:
+            return self.box(self._read(storage, i, offset))
 
     def read_bool(self, arr, i, offset):
-        return bool(self.for_computation(self._read(arr.storage, i, offset)))
+        with arr as storage:
+            return bool(self.for_computation(self._read(storage, i, offset)))
 
     def store(self, arr, i, offset, box):
-        self._write(arr.storage, i, offset, self.unbox(box))
+        with arr as storage:
+            self._write(storage, i, offset, self.unbox(box))
 
     def fill(self, storage, width, box, start, stop, offset):
         value = self.unbox(box)
@@ -1080,8 +1083,9 @@
         return bool(real) or bool(imag)
 
     def read_bool(self, arr, i, offset):
-        v = self.for_computation(self._read(arr.storage, i, offset))
-        return bool(v[0]) or bool(v[1])
+        with arr as storage:
+            v = self.for_computation(self._read(storage, i, offset))
+            return bool(v[0]) or bool(v[1])
 
     def get_element_size(self):
         return 2 * rffi.sizeof(self.T)
@@ -1132,8 +1136,9 @@
         return real, imag
 
     def read(self, arr, i, offset, dtype=None):
-        real, imag = self._read(arr.storage, i, offset)
-        return self.box_complex(real, imag)
+        with arr as storage:
+            real, imag = self._read(storage, i, offset)
+            return self.box_complex(real, imag)
 
     def _write(self, storage, i, offset, value):
         real, imag = value
@@ -1144,7 +1149,8 @@
         raw_storage_setitem_unaligned(storage, i + offset + rffi.sizeof(self.T), imag)
 

From noreply at buildbot.pypy.org  Wed Apr 22 12:20:48 2015
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Wed, 22 Apr 2015 12:20:48 +0200 (CEST)
Subject: [pypy-commit] pypy default: Merged pypy/pypy into default
Message-ID: <20150422102048.8ED911C06CD@cobra.cs.uni-duesseldorf.de>

Author: anton_gulenko 
Branch: 
Changeset: r76871:cc41fe9a9031
Date: 2015-03-12 13:07 +0100
http://bitbucket.org/pypy/pypy/changeset/cc41fe9a9031/

Log:	Merged pypy/pypy into default

diff --git a/lib_pypy/cffi.egg-info b/lib_pypy/cffi.egg-info
--- a/lib_pypy/cffi.egg-info
+++ b/lib_pypy/cffi.egg-info
@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: cffi
-Version: 0.9.0
+Version: 0.9.1
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "0.9.0"
-__version_info__ = (0, 9, 0)
+__version__ = "0.9.1"
+__version_info__ = (0, 9, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -2,13 +2,15 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rlib import rdynload
 
+VERSION = "0.9.1"
+
 
 class Module(MixedModule):
 
     appleveldefs = {
         }
     interpleveldefs = {
-        '__version__': 'space.wrap("0.9.0")',
+        '__version__': 'space.wrap("%s")' % VERSION,
 
         'load_library': 'libraryobj.load_library',
 
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3249,4 +3249,4 @@
 
 def test_version():
     # this test is here mostly for PyPy
-    assert __version__ == "0.9.0"
+    assert __version__ == "0.9.1"
diff --git a/pypy/module/_cffi_backend/test/test_file.py b/pypy/module/_cffi_backend/test/test_file.py
--- a/pypy/module/_cffi_backend/test/test_file.py
+++ b/pypy/module/_cffi_backend/test/test_file.py
@@ -15,3 +15,15 @@
             "Update test/_backend_test_c.py by copying it from "
             "https://bitbucket.org/cffi/cffi/raw/default/c/test_c.py "
             "and killing the import lines at the start")
+
+def test_egginfo_version():
+    from pypy.module._cffi_backend import VERSION
+    line = "Version: %s\n" % VERSION
+    eggfile = py.path.local(__file__).join('..', '..', '..', '..', '..',
+                                           'lib_pypy', 'cffi.egg-info')
+    assert line in eggfile.readlines()
+
+def test_app_version():
+    from pypy.module import _cffi_backend
+    from lib_pypy import cffi
+    assert _cffi_backend.VERSION == cffi.__version__
diff --git a/pypy/module/_random/interp_random.py b/pypy/module/_random/interp_random.py
--- a/pypy/module/_random/interp_random.py
+++ b/pypy/module/_random/interp_random.py
@@ -4,7 +4,7 @@
 from pypy.interpreter.typedef import TypeDef
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.baseobjspace import W_Root
-from rpython.rlib.rarithmetic import r_uint, intmask
+from rpython.rlib.rarithmetic import r_uint, intmask, widen
 from rpython.rlib import rbigint, rrandom, rstring
 
 
@@ -54,8 +54,8 @@
     def getstate(self, space):
         state = [None] * (rrandom.N + 1)
         for i in range(rrandom.N):
-            state[i] = space.newint(intmask(self._rnd.state[i]))
-        state[rrandom.N] = space.newint(self._rnd.index)
+            state[i] = space.wrap(widen(self._rnd.state[i]))
+        state[rrandom.N] = space.newlong(self._rnd.index)
         return space.newtuple(state)
 
     def setstate(self, space, w_state):
diff --git a/pypy/module/_random/test/test_random.py b/pypy/module/_random/test/test_random.py
--- a/pypy/module/_random/test/test_random.py
+++ b/pypy/module/_random/test/test_random.py
@@ -41,6 +41,17 @@
         # does not crash
         rnd1.setstate((-1, ) * 624 + (0, ))
 
+    def test_state_repr(self):
+        # since app-level jumpahead salts with repr(state),
+        # it is important the repr is consistent with cpython
+        import _random
+        rnd = _random.Random()
+        rnd.seed(1234)
+        state = rnd.getstate()
+        s = repr(state)
+        assert len(s) == 7956
+        assert s.count('L') == 625
+
     def test_seed(self):
         import _random, sys
         rnd = _random.Random()
diff --git a/pypy/module/_ssl/__init__.py b/pypy/module/_ssl/__init__.py
--- a/pypy/module/_ssl/__init__.py
+++ b/pypy/module/_ssl/__init__.py
@@ -51,6 +51,11 @@
 
         super(Module, cls).buildloaders()
 
+    def setup_after_space_initialization(self):
+        """NOT_RPYTHON"""
+        from pypy.module._ssl.interp_ssl import PWINFO_STORAGE
+        PWINFO_STORAGE.clear()
+
     def startup(self, space):
         from rpython.rlib.ropenssl import init_ssl
         init_ssl()
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -700,7 +700,8 @@
 
         @rgc.must_be_light_finalizer
         def __del__(self):
-            lltype.free(self.buffer, flavor='raw')
+            if self.buffer:
+                lltype.free(self.buffer, flavor='raw')
 
         def setlen(self, size, zero=False, overallocate=True):
             if size > 0:
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -1462,6 +1462,7 @@
     imag = GetSetProperty(W_NDimArray.descr_get_imag,
                           W_NDimArray.descr_set_imag),
     conj = interp2app(W_NDimArray.descr_conj),
+    conjugate = interp2app(W_NDimArray.descr_conj),
 
     argsort  = interp2app(W_NDimArray.descr_argsort),
     sort  = interp2app(W_NDimArray.descr_sort),
diff --git a/pypy/module/micronumpy/test/test_complex.py b/pypy/module/micronumpy/test/test_complex.py
--- a/pypy/module/micronumpy/test/test_complex.py
+++ b/pypy/module/micronumpy/test/test_complex.py
@@ -382,6 +382,7 @@
         assert np.conjugate(1+2j) == 1-2j
 
         eye2 = np.array([[1, 0], [0, 1]])
+        assert (eye2.conjugate() == eye2).all()
         x = eye2 + 1j * eye2
         for a, b in zip(np.conjugate(x), np.array([[ 1.-1.j,  0.-0.j], [ 0.-0.j,  1.-1.j]])):
             assert a[0] == b[0]
diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py b/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py
--- a/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py
@@ -165,7 +165,8 @@
         assert lib.sin(12.3) == math.sin(12.3)
         v = ffi.verifier
         ext = v.get_extension()
-        assert 'distutils.extension.Extension' in str(ext.__class__)
+        assert 'distutils.extension.Extension' in str(ext.__class__) or \
+               'setuptools.extension.Extension' in str(ext.__class__)
         assert ext.sources == [maybe_relative_path(v.sourcefilename)]
         assert ext.name == v.get_module_name()
         assert ext.define_macros == [('TEST_EXTENSION_OBJECT', '1')]
@@ -194,7 +195,8 @@
         assert lib.test1eoes(7.0) == 42.0
         v = ffi.verifier
         ext = v.get_extension()
-        assert 'distutils.extension.Extension' in str(ext.__class__)
+        assert 'distutils.extension.Extension' in str(ext.__class__) or \
+               'setuptools.extension.Extension' in str(ext.__class__)
         assert ext.sources == [maybe_relative_path(v.sourcefilename),
                                extra_source]
         assert ext.name == v.get_module_name()
diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py b/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py
--- a/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py
@@ -4,6 +4,9 @@
 import subprocess
 from pypy.module.test_lib_pypy.cffi_tests.udir import udir
 
+if sys.platform == 'win32':
+    py.test.skip('snippets do not run on win32')
+
 def create_venv(name):
     tmpdir = udir.join(name)
     try:
@@ -13,6 +16,20 @@
     except OSError as e:
         py.test.skip("Cannot execute virtualenv: %s" % (e,))
 
+    try:
+        deepcopy = os.symlink
+    except:
+        import shutil, errno
+        def deepcopy(src, dst):
+            try:
+                shutil.copytree(src, dst)
+            except OSError as e:
+                if e.errno in (errno.ENOTDIR, errno.EINVAL):
+                    shutil.copy(src, dst)
+                else:
+                    print 'got errno',e.errno,'not',errno.ENOTDIR
+                    raise
+
     site_packages = None
     for dirpath, dirnames, filenames in os.walk(str(tmpdir)):
         if os.path.basename(dirpath) == 'site-packages':
@@ -32,7 +49,7 @@
                 modules += ('ply',)   # needed for older versions of pycparser
         for module in modules:
             target = imp.find_module(module)[1]
-            os.symlink(target, os.path.join(site_packages,
+            deepcopy(target, os.path.join(site_packages,
                                             os.path.basename(target)))
     return tmpdir
 
@@ -51,7 +68,11 @@
     python_f.write(py.code.Source(python_snippet))
     try:
         os.chdir(str(SNIPPET_DIR.join(dirname)))
-        vp = str(venv_dir.join('bin/python'))
+        if os.name == 'nt':
+            bindir = 'Scripts'
+        else:
+            bindir = 'bin'
+        vp = str(venv_dir.join(bindir).join('python'))
         subprocess.check_call((vp, 'setup.py', 'clean'))
         subprocess.check_call((vp, 'setup.py', 'install'))
         subprocess.check_call((vp, str(python_f)))
diff --git a/pypy/objspace/fake/checkmodule.py b/pypy/objspace/fake/checkmodule.py
--- a/pypy/objspace/fake/checkmodule.py
+++ b/pypy/objspace/fake/checkmodule.py
@@ -10,6 +10,7 @@
         mod = __import__('pypy.module.%s' % modname, None, None, ['__doc__'])
         # force computation and record what we wrap
         module = mod.Module(space, W_Root())
+        module.setup_after_space_initialization()
         module.startup(space)
         for name in module.loaders:
             seeobj_w.append(module._load_lazily(space, name))
diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -12,7 +12,7 @@
     SomeBuiltin, SomePBC, SomeInteger, TLS, SomeUnicodeCodePoint,
     s_None, s_ImpossibleValue, SomeBool, SomeTuple,
     SomeImpossibleValue, SomeUnicodeString, SomeList, HarmlesslyBlocked,
-    SomeWeakRef, SomeByteArray, SomeConstantType)
+    SomeWeakRef, SomeByteArray, SomeConstantType, SomeProperty)
 from rpython.annotator.classdef import InstanceSource, ClassDef
 from rpython.annotator.listdef import ListDef, ListItem
 from rpython.annotator.dictdef import DictDef
@@ -301,6 +301,8 @@
                 s1 = self.immutablevalue(x1)
                 assert isinstance(s1, SomeInstance)
                 result = SomeWeakRef(s1.classdef)
+        elif tp is property:
+            return SomeProperty(x)
         elif ishashable(x) and x in BUILTIN_ANALYZERS:
             _module = getattr(x,"__module__","unknown")
             result = SomeBuiltin(BUILTIN_ANALYZERS[x], methodname="%s.%s" % (_module, x.__name__))
diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py
--- a/rpython/annotator/description.py
+++ b/rpython/annotator/description.py
@@ -482,6 +482,19 @@
                 self.all_enforced_attrs = []    # no attribute allowed
 
     def add_source_attribute(self, name, value, mixin=False):
+        if isinstance(value, property):
+            # special case for property object
+            if value.fget is not None:
+                newname = name + '__getter__'
+                func = func_with_new_name(value.fget, newname)
+                self.add_source_attribute(newname, func, mixin)
+            if value.fset is not None:
+                newname = name + '__setter__'
+                func = func_with_new_name(value.fset, newname)
+                self.add_source_attribute(newname, func, mixin)
+            self.classdict[name] = Constant(value)
+            return
+
         if isinstance(value, types.FunctionType):
             # for debugging
             if not hasattr(value, 'class_'):
diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py
--- a/rpython/annotator/model.py
+++ b/rpython/annotator/model.py
@@ -587,6 +587,19 @@
         return False
 
 
+class SomeProperty(SomeObject):
+    # used for union error only 
+    immutable = True
+    knowntype = type(property)
+
+    def __init__(self, prop):
+        self.fget = prop.fget
+        self.fset = prop.fset
+
+    def can_be_none(self):
+        return False
+
+
 s_None = SomeNone()
 s_Bool = SomeBool()
 s_Int = SomeInteger()
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -4326,12 +4326,81 @@
         assert isinstance(s, annmodel.SomeString)
         assert not s.can_be_none()
 
-    def test_nonnulify(self):
-        s = annmodel.SomeString(can_be_None=True).nonnulify()
-        assert s.can_be_None is True
-        assert s.no_nul is True
-        s = annmodel.SomeChar().nonnulify()
-        assert s.no_nul is True
+    def test_property_getter(self):
+        class O1(object):
+            def __init__(self, x):
+                self._x = x
+            @property
+            def x(self):
+                return self._x
+        def f(n):
+            o = O1(n)
+            return o.x + getattr(o, 'x')
+        a = self.RPythonAnnotator()
+        s = a.build_types(f, [int])
+        assert isinstance(s, annmodel.SomeInteger)
+        op = list(graphof(a, f).iterblocks())[0].operations
+        i = 0
+        c = 0
+        while i < len(op):
+            if op[i].opname == 'getattr':
+                c += 1
+                assert op[i].args[1].value == 'x__getter__'
+                i += 1
+                assert i < len(op) and op[i].opname == 'simple_call' and \
+                            op[i].args[0] == op[i - 1].result
+            i += 1
+        assert c == 2
+
+    def test_property_setter(self):
+        class O2(object):
+            def __init__(self):
+                self._x = 0
+            def set_x(self, v):
+                self._x = v
+            x = property(fset=set_x)
+        def f(n):
+            o = O2()
+            o.x = n
+            setattr(o, 'x', n)
+        a = self.RPythonAnnotator()
+        s = a.build_types(f, [int])
+        op = list(graphof(a, f).iterblocks())[0].operations
+        i = 0
+        c = 0
+        while i < len(op):
+            if op[i].opname == 'getattr':
+                c += 1
+                assert op[i].args[1].value == 'x__setter__'
+                i += 1
+                assert i < len(op) and op[i].opname == 'simple_call' and \
+                            op[i].args[0] == op[i - 1].result and len(op[i].args) == 2
+            i += 1
+        assert c == 2
+
+    def test_property_unionerr(self):
+        class O1(object):
+            def __init__(self, x):
+                self._x = x
+            @property
+            def x(self):
+                return self._x
+        class O2(O1):
+            def set_x(self, v):
+                self._x = v
+            x = property(fset=set_x)
+        def f1(n):
+            o = O2(n)
+            return o.x
+        def f2(n):
+            o = O2(n)
+            o.x = 20
+        a = self.RPythonAnnotator()
+        with py.test.raises(annmodel.UnionError) as exc:
+            a.build_types(f1, [int])
+        a = self.RPythonAnnotator()
+        with py.test.raises(annmodel.UnionError) as exc:
+            a.build_types(f2, [int])
 
 
 def g(n):
diff --git a/rpython/annotator/test/test_model.py b/rpython/annotator/test/test_model.py
--- a/rpython/annotator/test/test_model.py
+++ b/rpython/annotator/test/test_model.py
@@ -117,3 +117,11 @@
     assert s_int != SomeInteger()
     assert not_const(s_int) == SomeInteger()
     assert not_const(s_None) == s_None
+
+
+def test_nonnulify():
+    s = SomeString(can_be_None=True).nonnulify()
+    assert s.can_be_None is True
+    assert s.no_nul is True
+    s = SomeChar().nonnulify()
+    assert s.no_nul is True
diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -5,7 +5,7 @@
 from __future__ import absolute_import
 
 from rpython.flowspace.operation import op
-from rpython.flowspace.model import const
+from rpython.flowspace.model import const, Constant
 from rpython.annotator.model import (SomeObject, SomeInteger, SomeBool,
     SomeString, SomeChar, SomeList, SomeDict, SomeTuple, SomeImpossibleValue,
     SomeUnicodeCodePoint, SomeInstance, SomeBuiltin, SomeBuiltinMethod,
@@ -715,6 +715,50 @@
             op.simple_call(get_setslice.result, v_start, v_stop, v_iterable)]
 
 
+def _find_property_meth(s_obj, attr, meth):
+    result = []
+    for clsdef in s_obj.classdef.getmro():
+        dct = clsdef.classdesc.classdict
+        if attr not in dct:
+            continue
+        obj = dct[attr]
+        if (not isinstance(obj, Constant) or 
+                not isinstance(obj.value, property)):
+            return
+        result.append(getattr(obj.value, meth))
+    return result
+
+
+ at op.getattr.register_transform(SomeInstance)
+def getattr_SomeInstance(annotator, v_obj, v_attr):
+    s_attr = annotator.annotation(v_attr)
+    if not s_attr.is_constant() or not isinstance(s_attr.const, str):
+        return
+    attr = s_attr.const
+    getters = _find_property_meth(annotator.annotation(v_obj), attr, 'fget')
+    if getters:
+        if all(getters):
+            get_getter = op.getattr(v_obj, const(attr + '__getter__'))
+            return [get_getter, op.simple_call(get_getter.result)]
+        elif not any(getters):
+            raise AnnotatorError("Attribute %r is unreadable" % attr)
+
+
+ at op.setattr.register_transform(SomeInstance)
+def setattr_SomeInstance(annotator, v_obj, v_attr, v_value):
+    s_attr = annotator.annotation(v_attr)
+    if not s_attr.is_constant() or not isinstance(s_attr.const, str):
+        return
+    attr = s_attr.const
+    setters = _find_property_meth(annotator.annotation(v_obj), attr, 'fset')
+    if setters:
+        if all(setters):
+            get_setter = op.getattr(v_obj, const(attr + '__setter__'))
+            return [get_setter, op.simple_call(get_setter.result, v_value)]
+        elif not any(setters):
+            raise AnnotatorError("Attribute %r is unwritable" % attr)
+
+
 class __extend__(SomeBuiltin):
     def call(self, args, implicit_init=False):
         args_s, kwds = args.unpack()
diff --git a/rpython/doc/arm.rst b/rpython/doc/arm.rst
--- a/rpython/doc/arm.rst
+++ b/rpython/doc/arm.rst
@@ -160,5 +160,5 @@
 
 The gcrootfinder option is needed to work around `issue 1377`_ and the jit-backend works around `issue 1376`_
 
-.. _issue 1377: https://bugs.pypy.org/issue1377
-.. _issue 1376: https://bugs.pypy.org/issue1376
+.. _issue 1377: https://bitbucket.org/pypy/pypy/issue/1377
+.. _issue 1376: https://bitbucket.org/pypy/pypy/issue/1376
diff --git a/rpython/doc/jit/virtualizable.rst b/rpython/doc/jit/virtualizable.rst
--- a/rpython/doc/jit/virtualizable.rst
+++ b/rpython/doc/jit/virtualizable.rst
@@ -44,8 +44,14 @@
 virtualizable arrays that can be very confusing. Those will usually end
 up with a compile-time error (as opposed to strange behavior). The rules are:
 
+* A virtualizable array must be a fixed-size list.  After it is
+  initialized (e.g. in ``Frame.__init__``) you cannot resize it at all.
+  You cannot assign a different list to the field, or even pass around the
+  list.  You can only access ``frame.array[index]`` directly.
+
 * Each array access must be with a known positive index that cannot raise
-  an ``IndexError``. Using ``no = jit.hint(no, promote=True)`` might be useful
+  an ``IndexError``.
+  Using ``index = jit.hint(index, promote=True)`` might be useful
   to get a constant-number access. This is only safe if the index is actually
   constant or changing rarely within the context of the user's code.
 
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -772,15 +772,10 @@
 
     def FOR_ITER(self, target):
         w_iterator = self.peekvalue()
-        try:
-            w_nextitem = op.next(w_iterator).eval(self)
-            self.pushvalue(w_nextitem)
-        except Raise as e:
-            if self.exception_match(e.w_exc.w_type, const(StopIteration)):
-                self.popvalue()
-                return target
-            else:
-                raise
+        self.blockstack.append(IterBlock(self, target))
+        w_nextitem = op.next(w_iterator).eval(self)
+        self.blockstack.pop()
+        self.pushvalue(w_nextitem)
 
     def SETUP_LOOP(self, target):
         block = LoopBlock(self, target)
@@ -1333,6 +1328,16 @@
         ctx.last_exception = w_exc
         return self.handlerposition   # jump to the handler
 
+class IterBlock(ExceptBlock):
+    """A pseudo-block to catch the StopIteration inside FOR_ITER"""
+    def handle(self, ctx, unroller):
+        w_exc = unroller.w_exc
+        if ctx.exception_match(w_exc.w_type, const(StopIteration)):
+            ctx.popvalue()
+            return self.handlerposition
+        else:
+            return ctx.unroll(unroller)
+
 class FinallyBlock(FrameBlock):
     """A try:finally: block.  Stores the position of the exception handler."""
 
diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py
--- a/rpython/flowspace/operation.py
+++ b/rpython/flowspace/operation.py
@@ -555,7 +555,7 @@
     opname = 'next'
     arity = 1
     can_overflow = False
-    canraise = []
+    canraise = [StopIteration, RuntimeError]
     pyfunc = staticmethod(next)
 
     def eval(self, ctx):
@@ -571,9 +571,7 @@
                 else:
                     ctx.replace_in_stack(it, next_unroller)
                     return const(v)
-        w_item = ctx.do_op(self)
-        ctx.recorder.guessexception(ctx, StopIteration, RuntimeError)
-        return w_item
+        return HLOperation.eval(self, ctx)
 
 class GetAttr(SingleDispatchMixin, HLOperation):
     opname = 'getattr'
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -143,13 +143,16 @@
             return
         for v in list:
             if v in self.vable_array_vars:
+                vars = self.vable_array_vars[v]
+                (v_base, arrayfielddescr, arraydescr) = vars
                 raise AssertionError(
                     "A virtualizable array is passed around; it should\n"
                     "only be used immediately after being read.  Note\n"
                     "that a possible cause is indexing with an index not\n"
                     "known non-negative, or catching IndexError, or\n"
                     "not inlining at all (for tests: use listops=True).\n"
-                    "Occurred in: %r" % self.graph)
+                    "This is about: %r\n"
+                    "Occurred in: %r" % (arrayfielddescr, self.graph))
             # extra explanation: with the way things are organized in
             # rpython/rlist.py, the ll_getitem becomes a function call
             # that is typically meant to be inlined by the JIT, but
diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py
--- a/rpython/jit/codewriter/test/test_flatten.py
+++ b/rpython/jit/codewriter/test/test_flatten.py
@@ -999,6 +999,7 @@
         e = py.test.raises(AssertionError, self.encoding_test, f, [], "!",
                            transform=True)
         assert str(e.value).startswith("A virtualizable array is passed aroun")
+        assert "" in str(e.value)
 
     def test_vable_attribute_list_copied_around(self):
         class F:
@@ -1014,6 +1015,7 @@
         e = py.test.raises(AssertionError, self.encoding_test, f, [], "!",
                            transform=True)
         assert str(e.value).startswith("A virtualizable array is passed aroun")
+        assert "" in str(e.value)
 
 
 def check_force_cast(FROM, TO, operations, value):
diff --git a/rpython/translator/c/src/mem.c b/rpython/translator/c/src/mem.c
--- a/rpython/translator/c/src/mem.c
+++ b/rpython/translator/c/src/mem.c
@@ -29,6 +29,8 @@
 RPY_EXTERN
 void pypy_debug_alloc_stop(void *addr)
 {
+  if (!addr)
+	return;
   struct pypy_debug_alloc_s **p;
   for (p = &pypy_debug_alloc_list; *p; p = &((*p)->next))
     if ((*p)->addr == addr)

From noreply at buildbot.pypy.org  Wed Apr 22 12:20:49 2015
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Wed, 22 Apr 2015 12:20:49 +0200 (CEST)
Subject: [pypy-commit] pypy default: Renamed test.py
Message-ID: <20150422102049.B98F21C06CD@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: 
Changeset: r76872:400f8dc54f5d
Date: 2015-03-07 16:18 +0100
http://bitbucket.org/pypy/pypy/changeset/400f8dc54f5d/

Log:	Renamed test.py

diff --git a/rpython/rlib/rstrategies/test.py b/rpython/rlib/rstrategies/test_rstrategies.py
rename from rpython/rlib/rstrategies/test.py
rename to rpython/rlib/rstrategies/test_rstrategies.py

From noreply at buildbot.pypy.org  Wed Apr 22 12:20:50 2015
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Wed, 22 Apr 2015 12:20:50 +0200 (CEST)
Subject: [pypy-commit] pypy default: Moved test file into test/ directory.
Message-ID: <20150422102050.D67271C06CD@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: 
Changeset: r76873:e9e6ff0ec517
Date: 2015-03-07 17:01 +0100
http://bitbucket.org/pypy/pypy/changeset/e9e6ff0ec517/

Log:	Moved test file into test/ directory. Added underscore to some
	internal methods.

diff --git a/rpython/rlib/rstrategies/rstrategies.py b/rpython/rlib/rstrategies/rstrategies.py
--- a/rpython/rlib/rstrategies/rstrategies.py
+++ b/rpython/rlib/rstrategies/rstrategies.py
@@ -74,7 +74,7 @@
     
     def __init__(self, root_class, all_strategy_classes=None):
         if all_strategy_classes is None:
-            all_strategy_classes = self.collect_subclasses(root_class)
+            all_strategy_classes = self._collect_subclasses(root_class)
         self.strategies = []
         self.logger = logger.Logger()
         
@@ -82,15 +82,15 @@
         self.strategy_singleton_field = "__singleton_%i" % StrategyFactory.factory_instance_counter
         StrategyFactory.factory_instance_counter += 1
         
-        self.create_strategy_instances(root_class, all_strategy_classes)
+        self._create_strategy_instances(root_class, all_strategy_classes)
     
-    def create_strategy_instances(self, root_class, all_strategy_classes):
+    def _create_strategy_instances(self, root_class, all_strategy_classes):
         for strategy_class in all_strategy_classes:
             if strategy_class._is_strategy:
                 setattr(strategy_class, self.strategy_singleton_field, self.instantiate_strategy(strategy_class))
                 self.strategies.append(strategy_class)
-            self.patch_strategy_class(strategy_class, root_class)
-        self.order_strategies()
+            self._patch_strategy_class(strategy_class, root_class)
+        self._order_strategies()
     
     # =============================
     # API methods
@@ -217,7 +217,7 @@
     # Internal methods
     # =============================
     
-    def patch_strategy_class(self, strategy_class, root_class):
+    def _patch_strategy_class(self, strategy_class, root_class):
         "NOT_RPYTHON"
         # Patch root class: Add default handler for visitor
         def convert_storage_from_OTHER(self, w_self, previous_strategy):
@@ -231,15 +231,15 @@
             getattr(new_strategy, funcname)(w_self, self)
         strategy_class.convert_storage_to = convert_storage_to
     
-    def collect_subclasses(self, cls):
+    def _collect_subclasses(self, cls):
         "NOT_RPYTHON"
         subclasses = []
         for subcls in cls.__subclasses__():
             subclasses.append(subcls)
-            subclasses.extend(self.collect_subclasses(subcls))
+            subclasses.extend(self._collect_subclasses(subcls))
         return subclasses
     
-    def order_strategies(self):
+    def _order_strategies(self):
         "NOT_RPYTHON"
         def get_generalization_depth(strategy, visited=None):
             if visited is None:
@@ -327,7 +327,7 @@
         raise NotImplementedError("Abstract method")
     
     def convert_storage_to(self, w_self, new_strategy):
-        # This will be overwritten in patch_strategy_class
+        # This will be overwritten in _patch_strategy_class
         new_strategy.convert_storage_from(w_self, self)
     
     @jit.unroll_safe
diff --git a/rpython/rlib/rstrategies/test_rstrategies.py b/rpython/rlib/rstrategies/test/test_rstrategies.py
rename from rpython/rlib/rstrategies/test_rstrategies.py
rename to rpython/rlib/rstrategies/test/test_rstrategies.py

From noreply at buildbot.pypy.org  Wed Apr 22 12:20:52 2015
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Wed, 22 Apr 2015 12:20:52 +0200 (CEST)
Subject: [pypy-commit] pypy default: Added underscore to some more methods.
Message-ID: <20150422102052.06A651C06CD@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: 
Changeset: r76874:45166060bc6f
Date: 2015-03-07 17:10 +0100
http://bitbucket.org/pypy/pypy/changeset/45166060bc6f/

Log:	Added underscore to some more methods.

diff --git a/rpython/rlib/rstrategies/rstrategies.py b/rpython/rlib/rstrategies/rstrategies.py
--- a/rpython/rlib/rstrategies/rstrategies.py
+++ b/rpython/rlib/rstrategies/rstrategies.py
@@ -56,7 +56,7 @@
             def generalized_strategy_for(self, value):
                 # TODO - optimize this method
                 for strategy in generalize:
-                    if self.strategy_factory().strategy_singleton_instance(strategy).check_can_handle(value):
+                    if self.strategy_factory().strategy_singleton_instance(strategy)._check_can_handle(value):
                         return strategy
                 raise Exception("Could not find generalized strategy for %s coming from %s" % (value, self))
             strategy_class.generalized_strategy_for = generalized_strategy_for
@@ -109,7 +109,7 @@
             size = old_strategy.size(w_self)
             new_strategy = self.instantiate_strategy(new_strategy_type, w_self, size)
         self.set_strategy(w_self, new_strategy)
-        old_strategy.convert_storage_to(w_self, new_strategy)
+        old_strategy._convert_storage_to(w_self, new_strategy)
         new_strategy.strategy_switched(w_self)
         self.log(w_self, new_strategy, old_strategy, new_element)
         return new_strategy
@@ -127,7 +127,7 @@
         else:
             strategy = self.instantiate_strategy(strategy_type, w_self, size)
         self.set_strategy(w_self, strategy)
-        strategy.initialize_storage(w_self, size)
+        strategy._initialize_storage(w_self, size)
         element = None
         if elements:
             strategy.store_all(w_self, elements)
@@ -147,7 +147,7 @@
             if specialized_strategies <= 1:
                 break
             for i, strategy in enumerate(self.strategies):
-                if can_handle[i] and not self.strategy_singleton_instance(strategy).check_can_handle(obj):
+                if can_handle[i] and not self.strategy_singleton_instance(strategy)._check_can_handle(obj):
                     can_handle[i] = False
                     specialized_strategies -= 1
         for i, strategy_type in enumerate(self.strategies):
@@ -220,16 +220,16 @@
     def _patch_strategy_class(self, strategy_class, root_class):
         "NOT_RPYTHON"
         # Patch root class: Add default handler for visitor
-        def convert_storage_from_OTHER(self, w_self, previous_strategy):
-            self.convert_storage_from(w_self, previous_strategy)
-        funcname = "convert_storage_from_" + strategy_class.__name__
-        convert_storage_from_OTHER.func_name = funcname
-        setattr(root_class, funcname, convert_storage_from_OTHER)
+        def _convert_storage_from_OTHER(self, w_self, previous_strategy):
+            self._convert_storage_from(w_self, previous_strategy)
+        funcname = "_convert_storage_from_" + strategy_class.__name__
+        _convert_storage_from_OTHER.func_name = funcname
+        setattr(root_class, funcname, _convert_storage_from_OTHER)
         
         # Patch strategy class: Add polymorphic visitor function
-        def convert_storage_to(self, w_self, new_strategy):
+        def _convert_storage_to(self, w_self, new_strategy):
             getattr(new_strategy, funcname)(w_self, self)
-        strategy_class.convert_storage_to = convert_storage_to
+        strategy_class._convert_storage_to = _convert_storage_to
     
     def _collect_subclasses(self, cls):
         "NOT_RPYTHON"
@@ -320,37 +320,37 @@
 
     # Internal methods
     
-    def initialize_storage(self, w_self, initial_size):
+    def _initialize_storage(self, w_self, initial_size):
         raise NotImplementedError("Abstract method")
     
-    def check_can_handle(self, value):
+    def _check_can_handle(self, value):
         raise NotImplementedError("Abstract method")
     
-    def convert_storage_to(self, w_self, new_strategy):
+    def _convert_storage_to(self, w_self, new_strategy):
         # This will be overwritten in _patch_strategy_class
-        new_strategy.convert_storage_from(w_self, self)
+        new_strategy._convert_storage_from(w_self, self)
     
     @jit.unroll_safe
-    def convert_storage_from(self, w_self, previous_strategy):
+    def _convert_storage_from(self, w_self, previous_strategy):
         # This is a very unefficient (but most generic) way to do this.
         # Subclasses should specialize.
         storage = previous_strategy.fetch_all(w_self)
-        self.initialize_storage(w_self, previous_strategy.size(w_self))
+        self._initialize_storage(w_self, previous_strategy.size(w_self))
         for i, field in enumerate(storage):
             self.store(w_self, i, field)
     
-    def generalize_for_value(self, w_self, value):
+    def _generalize_for_value(self, w_self, value):
         strategy_type = self.generalized_strategy_for(value)
         new_instance = self.strategy_factory().switch_strategy(w_self, strategy_type, new_element=value)
         return new_instance
         
-    def cannot_handle_store(self, w_self, index0, value):
-        new_instance = self.generalize_for_value(w_self, value)
+    def _cannot_handle_store(self, w_self, index0, value):
+        new_instance = self._generalize_for_value(w_self, value)
         new_instance.store(w_self, index0, value)
         
-    def cannot_handle_insert(self, w_self, index0, list_w):
+    def _cannot_handle_insert(self, w_self, index0, list_w):
         # TODO - optimize. Prevent multiple generalizations and slicing done by callers.
-        new_strategy = self.generalize_for_value(w_self, list_w[0])
+        new_strategy = self._generalize_for_value(w_self, list_w[0])
         new_strategy.insert(w_self, index0, list_w)
 
 # ============== Special Strategies with no storage array ==============
@@ -359,23 +359,24 @@
     # == Required:
     # See AbstractStrategy
     
-    def initialize_storage(self, w_self, initial_size):
+    def _initialize_storage(self, w_self, initial_size):
         assert initial_size == 0
         self.set_storage(w_self, None)
-    def convert_storage_from(self, w_self, previous_strategy):
+    def _convert_storage_from(self, w_self, previous_strategy):
         self.set_storage(w_self, None)
+    def _check_can_handle(self, value):
+        return False
+    
     def fetch(self, w_self, index0):
         raise IndexError
     def store(self, w_self, index0, value):
-        self.cannot_handle_insert(w_self, index0, [value])
+        self._cannot_handle_insert(w_self, index0, [value])
     def insert(self, w_self, index0, list_w):
-        self.cannot_handle_insert(w_self, index0, list_w)
+        self._cannot_handle_insert(w_self, index0, list_w)
     def delete(self, w_self, start, end):
         self.check_index_range(w_self, start, end)
     def size(self, w_self):
         return 0
-    def check_can_handle(self, value):
-        return False
 
 class SingleValueStrategyStorage(object):
     """Small container object for a size value."""
@@ -389,39 +390,38 @@
     # check_index_*(...) - use mixin SafeIndexingMixin or UnsafeIndexingMixin
     # value(self) - the single value contained in this strategy. Should be constant.
     
-    def initialize_storage(self, w_self, initial_size):
+    def _initialize_storage(self, w_self, initial_size):
         storage_obj = SingleValueStrategyStorage(initial_size)
         self.set_storage(w_self, storage_obj)
-    def convert_storage_from(self, w_self, previous_strategy):
-        self.initialize_storage(w_self, previous_strategy.size(w_self))
+    def _convert_storage_from(self, w_self, previous_strategy):
+        self._initialize_storage(w_self, previous_strategy.size(w_self))
+    def _check_can_handle(self, value):
+        return value is self.value()
     
     def fetch(self, w_self, index0):
         self.check_index_fetch(w_self, index0)
         return self.value()
     def store(self, w_self, index0, value):
         self.check_index_store(w_self, index0)
-        if self.check_can_handle(value):
+        if self._check_can_handle(value):
             return
-        self.cannot_handle_store(w_self, index0, value)
+        self._cannot_handle_store(w_self, index0, value)
+    def delete(self, w_self, start, end):
+        self.check_index_range(w_self, start, end)
+        self.get_storage(w_self).size -= (end - start)
+    def size(self, w_self):
+        return self.get_storage(w_self).size
     
     @jit.unroll_safe
     def insert(self, w_self, index0, list_w):
         storage_obj = self.get_storage(w_self)
         for i in range(len(list_w)):
-            if self.check_can_handle(list_w[i]):
+            if self._check_can_handle(list_w[i]):
                 storage_obj.size += 1
             else:
-                self.cannot_handle_insert(w_self, index0 + i, list_w[i:])
+                self._cannot_handle_insert(w_self, index0 + i, list_w[i:])
                 return
-    
-    def delete(self, w_self, start, end):
-        self.check_index_range(w_self, start, end)
-        self.get_storage(w_self).size -= (end - start)
-    def size(self, w_self):
-        return self.get_storage(w_self).size
-    def check_can_handle(self, value):
-        return value is self.value()
-    
+
 # ============== Basic strategies with storage ==============
 
 class StrategyWithStorage(AbstractStrategy):
@@ -430,12 +430,12 @@
     # check_index_*(...) - use mixin SafeIndexingMixin or UnsafeIndexingMixin
     # default_value(self) - The value to be initially contained in this strategy
     
-    def initialize_storage(self, w_self, initial_size):
+    def _initialize_storage(self, w_self, initial_size):
         default = self._unwrap(self.default_value())
         self.set_storage(w_self, [default] * initial_size)
     
     @jit.unroll_safe
-    def convert_storage_from(self, w_self, previous_strategy):
+    def _convert_storage_from(self, w_self, previous_strategy):
         size = previous_strategy.size(w_self)
         new_storage = [ self._unwrap(previous_strategy.fetch(w_self, i))
                         for i in range(size) ]
@@ -443,11 +443,11 @@
     
     def store(self, w_self, index0, wrapped_value):
         self.check_index_store(w_self, index0)
-        if self.check_can_handle(wrapped_value):
+        if self._check_can_handle(wrapped_value):
             unwrapped = self._unwrap(wrapped_value)
             self.get_storage(w_self)[index0] = unwrapped
         else:
-            self.cannot_handle_store(w_self, index0, wrapped_value)
+            self._cannot_handle_store(w_self, index0, wrapped_value)
     
     def fetch(self, w_self, index0):
         self.check_index_fetch(w_self, index0)
@@ -468,10 +468,10 @@
         if start > self.size(w_self):
             start = self.size(w_self)
         for i in range(len(list_w)):
-            if self.check_can_handle(list_w[i]):
+            if self._check_can_handle(list_w[i]):
                 self.get_storage(w_self).insert(start + i, self._unwrap(list_w[i]))
             else:
-                self.cannot_handle_insert(w_self, start + i, list_w[i:])
+                self._cannot_handle_insert(w_self, start + i, list_w[i:])
                 return
     
     def delete(self, w_self, start, end):
@@ -487,7 +487,7 @@
         return value
     def _unwrap(self, value):
         return value
-    def check_can_handle(self, wrapped_value):
+    def _check_can_handle(self, wrapped_value):
         return True
     
 class WeakGenericStrategy(StrategyWithStorage):
@@ -499,7 +499,7 @@
     def _unwrap(self, value):
         assert value is not None
         return weakref.ref(value)
-    def check_can_handle(self, wrapped_value):
+    def _check_can_handle(self, wrapped_value):
         return True
     
 # ============== Mixins for index checking operations ==============
@@ -544,7 +544,7 @@
     # See SpecializedStrategy
     # contained_type - The wrapped type that can be stored in this strategy
     
-    def check_can_handle(self, value):
+    def _check_can_handle(self, value):
         return isinstance(value, self.contained_type)
     
 class TaggingStrategy(SingleTypeStrategy):
@@ -554,7 +554,7 @@
     # wrapped_tagged_value(self) - The tagged object
     # unwrapped_tagged_value(self) - The unwrapped tag value representing the tagged object
     
-    def check_can_handle(self, value):
+    def _check_can_handle(self, value):
         return value is self.wrapped_tagged_value() or \
                 (isinstance(value, self.contained_type) and \
                 self.unwrap(value) != self.unwrapped_tagged_value())
diff --git a/rpython/rlib/rstrategies/test/test_rstrategies.py b/rpython/rlib/rstrategies/test/test_rstrategies.py
--- a/rpython/rlib/rstrategies/test/test_rstrategies.py
+++ b/rpython/rlib/rstrategies/test/test_rstrategies.py
@@ -323,9 +323,9 @@
     def assert_handles(cls, good, bad):
         s = cls(0)
         for val in good:
-            assert s.check_can_handle(val)
+            assert s._check_can_handle(val)
         for val in bad:
-            assert not s.check_can_handle(val)
+            assert not s._check_can_handle(val)
     obj = W_Object()
     i = W_Integer(0)
     nil = w_nil
@@ -425,8 +425,8 @@
     def convert_storage_from_special(self, w_self, other):
         s.copied += 1
     
-    monkeypatch.setattr(AbstractStrategy, "convert_storage_from_NilStrategy", convert_storage_from_special)
-    monkeypatch.setattr(AbstractStrategy, "convert_storage_from", convert_storage_from_default)
+    monkeypatch.setattr(AbstractStrategy, "_convert_storage_from_NilStrategy", convert_storage_from_special)
+    monkeypatch.setattr(AbstractStrategy, "_convert_storage_from", convert_storage_from_default)
     try:
         factory.switch_strategy(l, IntegerOrNilStrategy)
     finally:

From noreply at buildbot.pypy.org  Wed Apr 22 12:20:53 2015
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Wed, 22 Apr 2015 12:20:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: Added tests to improve coverage.
 logger.py and rstrategies.py: 96%.
Message-ID: <20150422102053.2BE901C06CD@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: 
Changeset: r76875:0f889cf7f755
Date: 2015-03-12 13:04 +0100
http://bitbucket.org/pypy/pypy/changeset/0f889cf7f755/

Log:	Added tests to improve coverage. logger.py and rstrategies.py: 96%.

diff --git a/rpython/rlib/rstrategies/.coveragerc b/rpython/rlib/rstrategies/.coveragerc
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rstrategies/.coveragerc
@@ -0,0 +1,10 @@
+# .coveragerc file to control coverage.py (code coverage plugin for pytest)
+# Get it here: https://pypi.python.org/pypi/pytest-cov
+# Examples:
+# $ python -m pytest test --cov rpython.rlib.rstrategies --cov-report html --cov-config .coveragerc
+
+[run]
+omit =
+    test/*
+    */__init__.py
+    logparser.py
diff --git a/rpython/rlib/rstrategies/logger.py b/rpython/rlib/rstrategies/logger.py
--- a/rpython/rlib/rstrategies/logger.py
+++ b/rpython/rlib/rstrategies/logger.py
@@ -51,4 +51,8 @@
         classname_string = " of %s" % typename if typename else ""
         element_string = (" elements: " + " ".join(element_typenames)) if element_typenames else ""
         format = (cause, old_strategy_string, new_strategy, classname_string, slots, objects, element_string)
-        print "%s (%s%s)%s size %d objects %d%s" % format
+        self.do_print("%s (%s%s)%s size %d objects %d%s" % format)
+    
+    def do_print(self, str):
+        # Hook to increase testability
+        print str
diff --git a/rpython/rlib/rstrategies/test/test_rstrategies.py b/rpython/rlib/rstrategies/test/test_rstrategies.py
--- a/rpython/rlib/rstrategies/test/test_rstrategies.py
+++ b/rpython/rlib/rstrategies/test/test_rstrategies.py
@@ -72,11 +72,10 @@
     
     def __init__(self, root_class):
         self.decorate_strategies({
-            EmptyStrategy: [GenericStrategy],
+            EmptyStrategy: [NilStrategy, IntegerStrategy, IntegerOrNilStrategy, GenericStrategy],
             NilStrategy: [IntegerOrNilStrategy, GenericStrategy],
             GenericStrategy: [],
-            WeakGenericStrategy: [],
-                IntegerStrategy: [IntegerOrNilStrategy, GenericStrategy],
+            IntegerStrategy: [IntegerOrNilStrategy, GenericStrategy],
             IntegerOrNilStrategy: [GenericStrategy],
         })
         rs.StrategyFactory.__init__(self, root_class)
@@ -94,6 +93,7 @@
 
 class EmptyStrategy(AbstractStrategy):
     import_from_mixin(rs.EmptyStrategy)
+    # TODO - implement and test transition from Generic back to Empty
 
 class NilStrategy(AbstractStrategy):
     import_from_mixin(rs.SingleValueStrategy)
@@ -129,11 +129,16 @@
     def __init__(self, factory, w_list=None, size=0):
         super(NonSingletonStrategy, self).__init__(factory, w_list, size)
         self.w_list = w_list
-        self.size = size
+        self.the_size = size
 
 class NonStrategy(NonSingletonStrategy):
     pass
 
+ at rs.strategy(generalize=[])
+class InefficientStrategy(GenericStrategy):
+    def _convert_storage_from(self, w_self, previous_strategy):
+        return AbstractStrategy._convert_storage_from(self, w_self, previous_strategy)
+
 factory = Factory(AbstractStrategy)
 
 def check_contents(list, expected):
@@ -207,6 +212,8 @@
     assert s.size(l) == 0
     py.test.raises(IndexError, s.fetch, l, 0)
     py.test.raises(IndexError, s.fetch, l, 10)
+    py.test.raises(IndexError, s.delete, l, 0, 1)
+    py.test.raises(AssertionError, W_List, EmptyStrategy, 2) # Only size 0 possible.
     
 def test_init_Nil():
     do_test_initialization(NilStrategy)
@@ -265,13 +272,17 @@
 def do_test_insert(cls, values):
     l = W_List(cls, 0)
     assert len(values) >= 6
-    values1 = values[0:2]
+    values0 = values[0:1]
+    values1 = values[1:2]
     values2 = values[2:4]
     values3 = values[4:6]
-    l.insert(0, values1+values3)
-    check_contents(l, values1+values3)
+    l.insert(3, values0) # Will still be inserted at the very beginning
+    check_contents(l, values0)
+    l.insert(1, values1+values3)
+    check_contents(l, values0+values1+values3)
     l.insert(2, values2)
     check_contents(l, values)
+    return l
 
 def test_insert_Nil():
     do_test_insert(NilStrategy, [w_nil]*6)
@@ -291,9 +302,11 @@
     
 # === Test Delete
 
-def do_test_delete(cls, values):
+def do_test_delete(cls, values, indexing_unsafe=False):
     assert len(values) >= 6
     l = W_List(cls, len(values), values)
+    if not indexing_unsafe:
+        py.test.raises(IndexError, l.delete, 2, 1)
     l.delete(2, 4)
     del values[2: 4]
     check_contents(l, values)
@@ -305,7 +318,7 @@
     do_test_delete(NilStrategy, [w_nil]*6)
 
 def test_delete_Generic():
-    do_test_delete(GenericStrategy, [W_Object() for _ in range(6)])
+    do_test_delete(GenericStrategy, [W_Object() for _ in range(6)], indexing_unsafe=True)
     
 def test_delete_WeakGeneric():
     do_test_delete(WeakGenericStrategy, [W_Object() for _ in range(6)])
@@ -356,6 +369,13 @@
 def test_Integer_to_IntegerOrNil():
     do_test_transition(IntegerStrategy, w_nil, IntegerOrNilStrategy)
 
+def test_Generic_to_AllNil():
+    w = W_List(GenericStrategy, 5)
+    old = w.strategy
+    factory.switch_strategy(w, NilStrategy)
+    assert isinstance(w.strategy, NilStrategy)
+    assert factory.switching_log == [(None, old), (old, w.strategy)]
+
 def test_Integer_Generic():
     do_test_transition(IntegerStrategy, W_Object(), GenericStrategy)
 
@@ -363,7 +383,51 @@
     tag = IntegerOrNilStrategy(10).unwrapped_tagged_value() # sys.maxint
     do_test_transition(IntegerOrNilStrategy, W_Integer(tag), GenericStrategy)
 
-# TODO - Test transition from varsize back to Empty
+def test_insert_StrategySwitch_IntOrNil():
+    o = W_Object()
+    l = do_test_insert(IntegerOrNilStrategy, [W_Integer(1), w_nil, o, o, w_nil, W_Integer(3)])
+    assert isinstance(l.strategy, GenericStrategy)
+
+def test_insert_StrategySwitch_AllNil():
+    o = W_Object()
+    l = do_test_insert(NilStrategy, [w_nil, w_nil, o, o, w_nil, w_nil])
+    assert isinstance(l.strategy, GenericStrategy)
+    
+def test_transition_to_nonSingleton():
+    l = W_List(NilStrategy, 5)
+    factory.switch_strategy(l, NonSingletonStrategy)
+    strategy1 = l.strategy
+    assert isinstance(strategy1, NonSingletonStrategy)
+    factory.switch_strategy(l, NonSingletonStrategy)
+    assert strategy1 != l.strategy
+
+def test_generic_convert_storage():
+    l = W_List(NilStrategy, 5)
+    # This triggers AbstractStrategy._convert_storage_from
+    factory.switch_strategy(l, InefficientStrategy)
+    assert isinstance(l.strategy, InefficientStrategy)
+    assert l.fetch_all() == [w_nil] * 5
+
+def test_Empty_store():
+    l = W_List(EmptyStrategy, 0)
+    o = W_Object()
+    py.test.raises(IndexError, l.store, 0, o)
+    py.test.raises(IndexError, l.store, 1, o)
+
+def test_Empty_insert():
+    def do_insert(obj, expected_strategy, default_element=w_nil):
+        l = W_List(EmptyStrategy, 0)
+        l.insert(0, [obj])
+        assert l.size() == 1
+        assert isinstance(l.strategy, expected_strategy)
+        assert l.fetch_all() == [obj]
+        # Also test insert with too-high index
+        l = W_List(EmptyStrategy, 0)
+        l.insert(5, [obj])
+        assert l.fetch_all() == [obj]
+    do_insert(W_Object(), GenericStrategy)
+    do_insert(w_nil, NilStrategy)
+    do_insert(W_Integer(1), IntegerStrategy)
 
 # === Test helper methods
 
@@ -432,3 +496,57 @@
     finally:
         monkeypatch.undo()
     assert s.copied == 1, "Optimized switching routine not called exactly one time."
+
+def test_strategy_type_for(monkeypatch):
+    assert factory.strategy_type_for([w_nil, w_nil]) == NilStrategy
+    assert factory.strategy_type_for([W_Integer(2), W_Integer(1)]) == IntegerStrategy
+    assert factory.strategy_type_for([w_nil, W_Integer(2), w_nil]) == IntegerOrNilStrategy
+    assert factory.strategy_type_for([w_nil, W_Integer(2), W_Object()]) == GenericStrategy
+    assert factory.strategy_type_for([W_Integer(2), w_nil, W_Object()]) == GenericStrategy
+    assert factory.strategy_type_for([W_Object(), W_Integer(2), w_nil]) == GenericStrategy
+    assert factory.strategy_type_for([]) == EmptyStrategy
+    monkeypatch.setattr(GenericStrategy, '_check_can_handle', lambda self, o: False)
+    try:
+        py.test.raises(Exception, factory.strategy_type_for, [W_Object(), W_Object()])
+    finally:
+        monkeypatch.undo()
+
+# === Logger tests
+
+def test_logger(monkeypatch):
+    strings = []
+    def string_collector(str): strings.append(str)
+    factory.logger.activate()
+    monkeypatch.setattr(factory.logger, 'do_print', string_collector)
+    try:
+        W_List(EmptyStrategy, 0)
+        l = W_List(IntegerStrategy, 3)
+        l.store(1, w_nil)
+    finally:
+        monkeypatch.undo()
+        factory.logger.active = False
+    assert strings == [
+        'Created (EmptyStrategy) size 0 objects 1',
+        'Created (IntegerStrategy) size 3 objects 1',
+        'Switched (IntegerStrategy -> IntegerOrNilStrategy) size 3 objects 1 elements: W_Object']
+
+def test_aggregating_logger(monkeypatch):
+    strings = []
+    def string_collector(str): strings.append(str)
+    factory.logger.activate(aggregate = True)
+    monkeypatch.setattr(factory.logger, 'do_print', string_collector)
+    try:
+        W_List(EmptyStrategy, 0)
+        l = W_List(IntegerStrategy, 3)
+        l.store(1, w_nil)
+        factory.logger.print_aggregated_log()
+    finally:
+        monkeypatch.undo()
+        factory.logger.active = False
+    # Order of aggregated log entries is random.
+    strings.sort()
+    assert strings == [
+        'Created (EmptyStrategy) size 0 objects 1',
+        'Created (IntegerStrategy) size 3 objects 1',
+        'Switched (IntegerStrategy -> IntegerOrNilStrategy) size 3 objects 1 elements: W_Object']
+    
\ No newline at end of file

From noreply at buildbot.pypy.org  Wed Apr 22 12:20:54 2015
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Wed, 22 Apr 2015 12:20:54 +0200 (CEST)
Subject: [pypy-commit] pypy default: 2 small bugfixes, 1 comment
Message-ID: <20150422102054.56FDD1C06CD@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: 
Changeset: r76876:86b93d0474f4
Date: 2015-03-12 13:05 +0100
http://bitbucket.org/pypy/pypy/changeset/86b93d0474f4/

Log:	2 small bugfixes, 1 comment

diff --git a/rpython/rlib/rstrategies/rstrategies.py b/rpython/rlib/rstrategies/rstrategies.py
--- a/rpython/rlib/rstrategies/rstrategies.py
+++ b/rpython/rlib/rstrategies/rstrategies.py
@@ -250,7 +250,7 @@
                 visited.add(strategy)
                 depth = 0
                 for generalization in strategy._generalizations:
-                    other_depth = get_generalization_depth(generalization, visited)
+                    other_depth = get_generalization_depth(generalization, set(visited))
                     depth = max(depth, other_depth)
                 return depth + 1
             else:
@@ -370,7 +370,7 @@
     def fetch(self, w_self, index0):
         raise IndexError
     def store(self, w_self, index0, value):
-        self._cannot_handle_insert(w_self, index0, [value])
+        self._cannot_handle_store(w_self, index0, [value])
     def insert(self, w_self, index0, list_w):
         self._cannot_handle_insert(w_self, index0, list_w)
     def delete(self, w_self, start, end):
@@ -465,6 +465,8 @@
     
     @jit.unroll_safe
     def insert(self, w_self, start, list_w):
+        # This is following Python's behaviour - insert automatically
+        # happens at the beginning of an array, even if index is larger
         if start > self.size(w_self):
             start = self.size(w_self)
         for i in range(len(list_w)):

From noreply at buildbot.pypy.org  Wed Apr 22 12:20:56 2015
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Wed, 22 Apr 2015 12:20:56 +0200 (CEST)
Subject: [pypy-commit] pypy default: Merged from pypy/pypy
Message-ID: <20150422102056.69C0D1C06CD@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: 
Changeset: r76877:f26c00008203
Date: 2015-03-12 13:11 +0100
http://bitbucket.org/pypy/pypy/changeset/f26c00008203/

Log:	Merged from pypy/pypy

diff too long, truncating to 2000 out of 13267 lines

diff --git a/lib_pypy/audioop.py b/lib_pypy/audioop.py
--- a/lib_pypy/audioop.py
+++ b/lib_pypy/audioop.py
@@ -480,6 +480,7 @@
                             state_d, prev_i, cur_i,
                             weightA, weightB)
     result = ffi.buffer(rv)[:trim_index]
+    d = state_d[0]
     samps = zip(prev_i, cur_i)
     return (result, (d, tuple(samps)))
 
diff --git a/lib_pypy/cffi.egg-info b/lib_pypy/cffi.egg-info
--- a/lib_pypy/cffi.egg-info
+++ b/lib_pypy/cffi.egg-info
@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: cffi
-Version: 0.8.6+
+Version: 0.9.1
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "0.8.6+"
-__version_info__ = (0, 8, 6, "plus")
+__version__ = "0.9.1"
+__version_info__ = (0, 9, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/gdbm.py b/lib_pypy/gdbm.py
--- a/lib_pypy/gdbm.py
+++ b/lib_pypy/gdbm.py
@@ -20,9 +20,11 @@
 } datum;
 
 datum gdbm_fetch(void*, datum);
+datum pygdbm_fetch(void*, char*, int);
 int gdbm_delete(void*, datum);
 int gdbm_store(void*, datum, datum, int);
 int gdbm_exists(void*, datum);
+int pygdbm_exists(void*, char*, int);
 
 int gdbm_reorganize(void*);
 
@@ -37,19 +39,29 @@
 ''')
 
 try:
+    verify_code = '''
+    #include "gdbm.h"
+
+    static datum pygdbm_fetch(GDBM_FILE gdbm_file, char *dptr, int dsize) {
+        datum key = {dptr, dsize};
+        return gdbm_fetch(gdbm_file, key);
+    }
+
+    static int pygdbm_exists(GDBM_FILE gdbm_file, char *dptr, int dsize) {
+        datum key = {dptr, dsize};
+        return gdbm_exists(gdbm_file, key);
+    }
+    
+    '''
     if sys.platform.startswith('freebsd'):
         import os.path
         _localbase = os.environ.get('LOCALBASE', '/usr/local')
-        lib = ffi.verify('''
-        #include "gdbm.h"
-        ''', libraries=['gdbm'],
+        lib = ffi.verify(verify_code, libraries=['gdbm'],
              include_dirs=[os.path.join(_localbase, 'include')],
              library_dirs=[os.path.join(_localbase, 'lib')]
         )
     else:
-        lib = ffi.verify('''
-        #include "gdbm.h"
-        ''', libraries=['gdbm'])
+        lib = ffi.verify(verify_code, libraries=['gdbm'])
 except cffi.VerificationError as e:
     # distutils does not preserve the actual message,
     # but the verification is simple enough that the
@@ -59,6 +71,13 @@
 class error(Exception):
     pass
 
+def _checkstr(key):
+    if isinstance(key, unicode):
+        key = key.encode("ascii")
+    if not isinstance(key, str):
+        raise TypeError("gdbm mappings have string indices only")
+    return key
+
 def _fromstr(key):
     if isinstance(key, unicode):
         key = key.encode("ascii")
@@ -107,12 +126,14 @@
 
     def __contains__(self, key):
         self._check_closed()
-        return lib.gdbm_exists(self.ll_dbm, _fromstr(key))
+        key = _checkstr(key)
+        return lib.pygdbm_exists(self.ll_dbm, key, len(key))
     has_key = __contains__
 
     def __getitem__(self, key):
         self._check_closed()
-        drec = lib.gdbm_fetch(self.ll_dbm, _fromstr(key))
+        key = _checkstr(key)        
+        drec = lib.pygdbm_fetch(self.ll_dbm, key, len(key))
         if not drec.dptr:
             raise KeyError(key)
         res = str(ffi.buffer(drec.dptr, drec.dsize))
diff --git a/lib_pypy/pyrepl/readline.py b/lib_pypy/pyrepl/readline.py
--- a/lib_pypy/pyrepl/readline.py
+++ b/lib_pypy/pyrepl/readline.py
@@ -73,7 +73,6 @@
     assume_immutable_completions = False
     use_brackets = False
     sort_in_column = True
-    tab_insert_spaces_if_stem_is_empty = False
 
     def error(self, msg="none"):
         pass    # don't show error messages by default
@@ -87,7 +86,7 @@
         return ''.join(b[p+1:self.pos])
 
     def get_completions(self, stem):
-        if len(stem) == 0 and self.tab_insert_spaces_if_stem_is_empty:
+        if len(stem) == 0 and self.more_lines is not None:
             b = self.buffer
             p = self.pos
             while p > 0 and b[p - 1] != '\n':
@@ -141,12 +140,16 @@
 
     def collect_keymap(self):
         return super(ReadlineAlikeReader, self).collect_keymap() + (
-            (r'\n', 'maybe-accept'),)
+            (r'\n', 'maybe-accept'),
+            (r'\', 'backspace-dedent'),
+            )
 
     def __init__(self, console):
         super(ReadlineAlikeReader, self).__init__(console)
         self.commands['maybe_accept'] = maybe_accept
         self.commands['maybe-accept'] = maybe_accept
+        self.commands['backspace_dedent'] = backspace_dedent
+        self.commands['backspace-dedent'] = backspace_dedent
 
     def after_command(self, cmd):
         super(ReadlineAlikeReader, self).after_command(cmd)
@@ -164,6 +167,28 @@
                 if self.pos > len(self.buffer):
                     self.pos = len(self.buffer)
 
+def _get_this_line_indent(buffer, pos):
+    indent = 0
+    while pos > 0 and buffer[pos - 1] in " \t":
+        indent += 1
+        pos -= 1
+    if pos > 0 and buffer[pos - 1] == "\n":
+        return indent
+    return 0
+
+def _get_previous_line_indent(buffer, pos):
+    prevlinestart = pos
+    while prevlinestart > 0 and buffer[prevlinestart - 1] != "\n":
+        prevlinestart -= 1
+    prevlinetext = prevlinestart
+    while prevlinetext < pos and buffer[prevlinetext] in " \t":
+        prevlinetext += 1
+    if prevlinetext == pos:
+        indent = None
+    else:
+        indent = prevlinetext - prevlinestart
+    return prevlinestart, indent
+
 class maybe_accept(commands.Command):
     def do(self):
         r = self.reader
@@ -172,13 +197,39 @@
         # if there are already several lines and the cursor
         # is not on the last one, always insert a new \n.
         text = r.get_unicode()
-        if "\n" in r.buffer[r.pos:]:
+        if ("\n" in r.buffer[r.pos:] or
+            (r.more_lines is not None and r.more_lines(text))):
+            #
+            # auto-indent the next line like the previous line
+            prevlinestart, indent = _get_previous_line_indent(r.buffer, r.pos)
             r.insert("\n")
-        elif r.more_lines is not None and r.more_lines(text):
-            r.insert("\n")
+            if indent:
+                for i in range(prevlinestart, prevlinestart + indent):
+                    r.insert(r.buffer[i])
         else:
             self.finish = 1
 
+class backspace_dedent(commands.Command):
+    def do(self):
+        r = self.reader
+        b = r.buffer
+        if r.pos > 0:
+            repeat = 1
+            if b[r.pos - 1] != "\n":
+                indent = _get_this_line_indent(b, r.pos)
+                if indent > 0:
+                    ls = r.pos - indent
+                    while ls > 0:
+                        ls, pi = _get_previous_line_indent(b, ls - 1)
+                        if pi is not None and pi < indent:
+                            repeat = indent - pi
+                            break
+            r.pos -= repeat
+            del b[r.pos:r.pos + repeat]
+            r.dirty = 1
+        else:
+            self.reader.error("can't backspace at start")
+
 # ____________________________________________________________
 
 class _ReadlineWrapper(object):
@@ -212,15 +263,14 @@
         boolean value is true.
         """
         reader = self.get_reader()
-        saved = reader.more_lines, reader.tab_insert_spaces_if_stem_is_empty
+        saved = reader.more_lines
         try:
             reader.more_lines = more_lines
             reader.ps1 = reader.ps2 = ps1
             reader.ps3 = reader.ps4 = ps2
-            reader.tab_insert_spaces_if_stem_is_empty = True
             return reader.readline(returns_unicode=returns_unicode)
         finally:
-            reader.more_lines, reader.tab_insert_spaces_if_stem_is_empty = saved
+            reader.more_lines = saved
 
     def parse_and_bind(self, string):
         pass  # XXX we don't support parsing GNU-readline-style init files
diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -36,7 +36,7 @@
 def pytest_addoption(parser):
     from rpython.conftest import pytest_addoption
     pytest_addoption(parser)
-    
+
     group = parser.getgroup("pypy options")
     group.addoption('-A', '--runappdirect', action="store_true",
            default=False, dest="runappdirect",
@@ -44,6 +44,9 @@
     group.addoption('--direct', action="store_true",
            default=False, dest="rundirect",
            help="run pexpect tests directly")
+    group.addoption('--raise-operr', action="store_true",
+            default=False, dest="raise_operr",
+            help="Show the interp-level OperationError in app-level tests")
 
 def pytest_funcarg__space(request):
     from pypy.tool.pytest.objspace import gettestobjspace
diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -36,7 +36,8 @@
    "PyPy home directory".  The arguments are:
 
    * ``home``: NULL terminated path to an executable inside the pypy directory
-     (can be a .so name, can be made up)
+     (can be a .so name, can be made up).  Used to look up the standard
+     library, and is also set as ``sys.executable``.
 
    * ``verbose``: if non-zero, it will print error messages to stderr
 
diff --git a/pypy/doc/jit-hooks.rst b/pypy/doc/jit-hooks.rst
--- a/pypy/doc/jit-hooks.rst
+++ b/pypy/doc/jit-hooks.rst
@@ -39,3 +39,30 @@
     Reason is a string, the meaning of other arguments is the same
     as attributes on JitLoopInfo object
 
+.. function:: enable_debug()
+
+    Start recording debugging counters for ``get_stats_snapshot``
+
+.. function:: disable_debug()
+
+    Stop recording debugging counters for ``get_stats_snapshot``
+
+.. function:: get_stats_snapshot()
+
+    Get the jit status in the specific moment in time. Note that this
+    is eager - the attribute access is not lazy, if you need new stats
+    you need to call this function again. You might want to call
+    ``enable_debug`` to get more information. It returns an instance
+    of ``JitInfoSnapshot``
+
+.. class:: JitInfoSnapshot
+
+    A class describing current snapshot. Usable attributes:
+
+    * ``counters`` - internal JIT integer counters
+
+    * ``counter_times`` - internal JIT float counters, notably time spent
+      TRACING and in the JIT BACKEND
+
+    * ``loop_run_times`` - counters for number of times loops are run, only
+      works when ``enable_debug`` is called.
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -6,8 +6,8 @@
 .. startrev: 397b96217b85
 
 
-Fix non-blocking file reads sometimes raising EAGAIN even though they
-have buffered data waiting (b1c4fcb04a42)
+Non-blocking file reads sometimes raised EAGAIN even though they
+had buffered data waiting, fixed in b1c4fcb04a42
 
 
 .. branch: vmprof
@@ -18,3 +18,22 @@
 
 .. branch: stdlib-2.7.9
 Update stdlib to version 2.7.9
+
+.. branch: fix-kqueue-error2
+Fix exception being raised by kqueue.control (CPython compatibility)
+
+.. branch: gitignore
+
+.. branch: framestate2
+Refactor rpython.flowspace.framestate.FrameState.
+
+.. branch: alt_errno
+Add an alternative location to save LastError, errno around ctypes,
+cffi external calls so things like pdb will not overwrite it
+
+.. branch: nonquadratic-heapcache
+Speed up the warmup times of the JIT by removing a quadratic algorithm in the
+heapcache.
+
+.. branch: online-transforms-2
+Simplify flow graphs on the fly during annotation phase.
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -106,6 +106,9 @@
         space.call_function(w_pathsetter, w_path)
         # import site
         try:
+            space.setattr(space.getbuiltinmodule('sys'),
+                          space.wrap('executable'),
+                          space.wrap(home))
             import_ = space.getattr(space.getbuiltinmodule('__builtin__'),
                                     space.wrap('__import__'))
             space.call_function(import_, space.wrap('site'))
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -1,5 +1,5 @@
 #! /usr/bin/env python
-# App-level version of py.py.
+# This is pure Python code that handles the main entry point into "pypy".
 # See test/test_app_main.
 
 # Missing vs CPython: -d, -t, -v, -x, -3
@@ -157,10 +157,13 @@
             current = group
     raise SystemExit
 
+def get_sys_executable():
+    return getattr(sys, 'executable', 'pypy')
+
 def print_help(*args):
     import os
     print 'usage: %s [option] ... [-c cmd | -m mod | file | -] [arg] ...' % (
-        sys.executable,)
+        get_sys_executable(),)
     print USAGE1,
     if 'pypyjit' in sys.builtin_module_names:
         print "--jit options: advanced JIT options: try 'off' or 'help'"
@@ -171,7 +174,7 @@
     try:
         import pypyjit
     except ImportError:
-        print >> sys.stderr, "No jit support in %s" % (sys.executable,)
+        print >> sys.stderr, "No jit support in %s" % (get_sys_executable(),)
         return
     items = sorted(pypyjit.defaults.items())
     print 'Advanced JIT options: a comma-separated list of OPTION=VALUE:'
@@ -209,7 +212,7 @@
         raise SystemExit
     if 'pypyjit' not in sys.builtin_module_names:
         print >> sys.stderr, ("Warning: No jit support in %s" %
-                              (sys.executable,))
+                              (get_sys_executable(),))
     else:
         import pypyjit
         pypyjit.set_param(jitparam)
@@ -219,8 +222,8 @@
 
 def print_error(msg):
     print >> sys.stderr, msg
-    print >> sys.stderr, 'usage: %s [options]' % (sys.executable,)
-    print >> sys.stderr, 'Try `%s -h` for more information.' % (sys.executable,)
+    print >> sys.stderr, 'usage: %s [options]' % (get_sys_executable(),)
+    print >> sys.stderr, 'Try `%s -h` for more information.' % (get_sys_executable(),)
 
 def fdopen(fd, mode, bufsize=-1):
     try:
@@ -514,6 +517,10 @@
     elif not sys.stdout.isatty():
         set_fully_buffered_io()
 
+    if we_are_translated():
+        import __pypy__
+        __pypy__.save_module_content_for_future_reload(sys)
+
     mainmodule = type(sys)('__main__')
     sys.modules['__main__'] = mainmodule
 
diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py
--- a/pypy/interpreter/mixedmodule.py
+++ b/pypy/interpreter/mixedmodule.py
@@ -55,7 +55,10 @@
         if self.w_initialdict is None:
             Module.init(self, space)
             if not self.lazy and self.w_initialdict is None:
-                self.w_initialdict = space.call_method(self.w_dict, 'items')
+                self.save_module_content_for_future_reload()
+
+    def save_module_content_for_future_reload(self):
+        self.w_initialdict = self.space.call_method(self.w_dict, 'items')
 
 
     def get_applevel_name(cls):
@@ -119,7 +122,7 @@
                 w_value = self.get(name)
                 space.setitem(self.w_dict, space.new_interned_str(name), w_value)
             self.lazy = False
-            self.w_initialdict = space.call_method(self.w_dict, 'items')
+            self.save_module_content_for_future_reload()
         return self.w_dict
 
     def _cleanup_(self):
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -373,7 +373,7 @@
         config = make_config(None)
         space = make_objspace(config)
         w_executable = space.wrap('executable')
-        assert space.str_w(space.getattr(space.sys, w_executable)) == 'py.py'
+        assert space.findattr(space.sys, w_executable) is None
         space.setattr(space.sys, w_executable, space.wrap('foobar'))
         assert space.str_w(space.getattr(space.sys, w_executable)) == 'foobar'
         space.startup()
diff --git a/pypy/interpreter/test/test_targetpypy.py b/pypy/interpreter/test/test_targetpypy.py
--- a/pypy/interpreter/test/test_targetpypy.py
+++ b/pypy/interpreter/test/test_targetpypy.py
@@ -8,7 +8,7 @@
         entry_point = get_entry_point(config)[0]
         entry_point(['pypy-c' , '-S', '-c', 'print 3'])
 
-def test_exeucte_source(space):
+def test_execute_source(space):
     _, d = create_entry_point(space, None)
     execute_source = d['pypy_execute_source']
     lls = rffi.str2charp("import sys; sys.modules['xyz'] = 3")
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -25,8 +25,8 @@
         self.reason = reason
 
 @specialize.memo()
-def encode_error_handler(space):
-    # Fast version of the "strict" errors handler.
+def rpy_encode_error_handler():
+    # A RPython version of the "strict" error handler.
     def raise_unicode_exception_encode(errors, encoding, msg, u,
                                        startingpos, endingpos):
         raise RUnicodeEncodeError(encoding, u, startingpos, endingpos, msg)
@@ -62,7 +62,10 @@
     return result
 
 def encode_utf8(space, uni):
+    # Note that this function never raises UnicodeEncodeError,
+    # since surrogate pairs are allowed.
+    # This is not the case with Python3.
     return runicode.unicode_encode_utf_8(
         uni, len(uni), "strict",
-        errorhandler=encode_error_handler(space),
+        errorhandler=rpy_encode_error_handler(),
         allow_surrogates=True)
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -82,6 +82,8 @@
         'strategy'                  : 'interp_magic.strategy',  # dict,set,list
         'set_debug'                 : 'interp_magic.set_debug',
         'locals_to_fast'            : 'interp_magic.locals_to_fast',
+        'save_module_content_for_future_reload':
+                          'interp_magic.save_module_content_for_future_reload',
     }
     if sys.platform == 'win32':
         interpleveldefs['get_console_cp'] = 'interp_magic.get_console_cp'
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -1,6 +1,7 @@
 from pypy.interpreter.error import OperationError, wrap_oserror
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.pyframe import PyFrame
+from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rlib.objectmodel import we_are_translated
 from pypy.objspace.std.dictmultiobject import W_DictMultiObject
 from pypy.objspace.std.listobject import W_ListObject
@@ -130,3 +131,7 @@
 def locals_to_fast(space, w_frame):
     assert isinstance(w_frame, PyFrame)
     w_frame.locals2fast()
+
+ at unwrap_spec(w_module=MixedModule)
+def save_module_content_for_future_reload(space, w_module):
+    w_module.save_module_content_for_future_reload()
diff --git a/pypy/module/__pypy__/test/test_magic.py b/pypy/module/__pypy__/test/test_magic.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__pypy__/test/test_magic.py
@@ -0,0 +1,15 @@
+
+class AppTestMagic:
+    spaceconfig = dict(usemodules=['__pypy__'])
+
+    def test_save_module_content_for_future_reload(self):
+        import sys, __pypy__
+        d = sys.dont_write_bytecode
+        sys.dont_write_bytecode = "hello world"
+        __pypy__.save_module_content_for_future_reload(sys)
+        sys.dont_write_bytecode = d
+        reload(sys)
+        assert sys.dont_write_bytecode == "hello world"
+        #
+        sys.dont_write_bytecode = d
+        __pypy__.save_module_content_for_future_reload(sys)
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -2,13 +2,15 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rlib import rdynload
 
+VERSION = "0.9.1"
+
 
 class Module(MixedModule):
 
     appleveldefs = {
         }
     interpleveldefs = {
-        '__version__': 'space.wrap("0.8.6+")',
+        '__version__': 'space.wrap("%s")' % VERSION,
 
         'load_library': 'libraryobj.load_library',
 
diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -81,4 +81,5 @@
     if size < 0:
         raise oefmt(space.w_TypeError,
                     "don't know the size pointed to by '%s'", ctype.name)
-    return space.wrap(MiniBuffer(LLBuffer(w_cdata._cdata, size), w_cdata))
+    ptr = w_cdata.unsafe_escaping_ptr()    # w_cdata kept alive by MiniBuffer()
+    return space.wrap(MiniBuffer(LLBuffer(ptr, size), w_cdata))
diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py
--- a/pypy/module/_cffi_backend/ccallback.py
+++ b/pypy/module/_cffi_backend/ccallback.py
@@ -48,9 +48,12 @@
             raise oefmt(space.w_NotImplementedError,
                         "%s: callback with unsupported argument or "
                         "return type or with '...'", self.getfunctype().name)
-        res = clibffi.c_ffi_prep_closure(self.get_closure(), cif_descr.cif,
-                                         invoke_callback,
-                                         rffi.cast(rffi.VOIDP, self.unique_id))
+        with self as ptr:
+            closure_ptr = rffi.cast(clibffi.FFI_CLOSUREP, ptr)
+            unique_id = rffi.cast(rffi.VOIDP, self.unique_id)
+            res = clibffi.c_ffi_prep_closure(closure_ptr, cif_descr.cif,
+                                             invoke_callback,
+                                             unique_id)
         if rffi.cast(lltype.Signed, res) != clibffi.FFI_OK:
             raise OperationError(space.w_SystemError,
                 space.wrap("libffi failed to build this callback"))
@@ -62,12 +65,9 @@
             from pypy.module.thread.os_thread import setup_threads
             setup_threads(space)
 
-    def get_closure(self):
-        return rffi.cast(clibffi.FFI_CLOSUREP, self._cdata)
-
     #@rgc.must_be_light_finalizer
     def __del__(self):
-        clibffi.closureHeap.free(self.get_closure())
+        clibffi.closureHeap.free(rffi.cast(clibffi.FFI_CLOSUREP, self._ptr))
         if self.ll_error:
             lltype.free(self.ll_error, flavor='raw')
 
@@ -106,7 +106,7 @@
         fresult = self.getfunctype().ctitem
         if fresult.size > 0:
             misc._raw_memcopy(self.ll_error, ll_res, fresult.size)
-            keepalive_until_here(self)
+            keepalive_until_here(self)     # to keep self.ll_error alive
 
 
 global_callback_mapping = rweakref.RWeakValueDictionary(int, W_CDataCallback)
@@ -210,6 +210,6 @@
         space.threadlocals.leave_thread(space)
 
 def invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata):
-    cerrno._errno_after(rffi.RFFI_ERR_ALL)
+    cerrno._errno_after(rffi.RFFI_ERR_ALL | rffi.RFFI_ALT_ERRNO)
     _invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata)
-    cerrno._errno_before(rffi.RFFI_ERR_ALL)
+    cerrno._errno_before(rffi.RFFI_ERR_ALL | rffi.RFFI_ALT_ERRNO)
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -14,21 +14,37 @@
 
 
 class W_CData(W_Root):
-    _attrs_ = ['space', '_cdata', 'ctype', '_lifeline_']
-    _immutable_fields_ = ['_cdata', 'ctype']
-    _cdata = lltype.nullptr(rffi.CCHARP.TO)
+    _attrs_ = ['space', '_ptr', 'ctype', '_lifeline_']
+    _immutable_fields_ = ['_ptr', 'ctype']
+    _ptr = lltype.nullptr(rffi.CCHARP.TO)
 
-    def __init__(self, space, cdata, ctype):
+    def __init__(self, space, ptr, ctype):
         from pypy.module._cffi_backend import ctypeobj
-        assert lltype.typeOf(cdata) == rffi.CCHARP
+        assert lltype.typeOf(ptr) == rffi.CCHARP
         assert isinstance(ctype, ctypeobj.W_CType)
         self.space = space
-        self._cdata = cdata    # don't forget keepalive_until_here!
+        self._ptr = ptr    # don't access directly!  use "with cdata as ptr:"
         self.ctype = ctype
 
+    def __enter__(self):
+        """Use 'with cdata as ptr:' to access the raw memory.  It will
+        stay alive at least until the end of the 'with' block.
+        """
+        return self._ptr
+
+    def __exit__(self, *args):
+        keepalive_until_here(self)
+
+    def unsafe_escaping_ptr(self):
+        """Generally unsafe: escape the pointer to raw memory.
+        If 'self' is a subclass that frees the pointer in a destructor,
+        it may be freed under your feet at any time.
+        """
+        return self._ptr
+
     def _repr_extra(self):
-        extra = self.ctype.extra_repr(self._cdata)
-        keepalive_until_here(self)
+        with self as ptr:
+            extra = self.ctype.extra_repr(ptr)
         return extra
 
     def _repr_extra_owning(self):
@@ -54,11 +70,13 @@
             self.ctype.name, extra1, extra2))
 
     def nonzero(self):
-        return self.space.wrap(bool(self._cdata))
+        with self as ptr:
+            nonzero = bool(ptr)
+        return self.space.wrap(nonzero)
 
     def int(self, space):
-        w_result = self.ctype.cast_to_int(self._cdata)
-        keepalive_until_here(self)
+        with self as ptr:
+            w_result = self.ctype.cast_to_int(ptr)
         return w_result
 
     def long(self, space):
@@ -69,8 +87,8 @@
         return w_result
 
     def float(self):
-        w_result = self.ctype.float(self._cdata)
-        keepalive_until_here(self)
+        with self as ptr:
+            w_result = self.ctype.float(ptr)
         return w_result
 
     def len(self):
@@ -88,20 +106,19 @@
         def _cmp(self, w_other):
             from pypy.module._cffi_backend.ctypeprim import W_CTypePrimitive
             space = self.space
-            cdata1 = self._cdata
-            if isinstance(w_other, W_CData):
-                cdata2 = w_other._cdata
-            else:
+            if not isinstance(w_other, W_CData):
                 return space.w_NotImplemented
 
-            if requires_ordering:
-                if (isinstance(self.ctype, W_CTypePrimitive) or
-                    isinstance(w_other.ctype, W_CTypePrimitive)):
-                    raise OperationError(space.w_TypeError,
-                       space.wrap("cannot do comparison on a primitive cdata"))
-                cdata1 = rffi.cast(lltype.Unsigned, cdata1)
-                cdata2 = rffi.cast(lltype.Unsigned, cdata2)
-            return space.newbool(op(cdata1, cdata2))
+            with self as ptr1, w_other as ptr2:
+                if requires_ordering:
+                    if (isinstance(self.ctype, W_CTypePrimitive) or
+                        isinstance(w_other.ctype, W_CTypePrimitive)):
+                        raise OperationError(space.w_TypeError, space.wrap(
+                            "cannot do comparison on a primitive cdata"))
+                    ptr1 = rffi.cast(lltype.Unsigned, ptr1)
+                    ptr2 = rffi.cast(lltype.Unsigned, ptr2)
+                result = op(ptr1, ptr2)
+            return space.newbool(result)
         #
         return func_with_new_name(_cmp, name)
 
@@ -113,7 +130,8 @@
     ge = _make_comparison('ge')
 
     def hash(self):
-        h = rffi.cast(lltype.Signed, self._cdata)
+        ptr = self.unsafe_escaping_ptr()
+        h = rffi.cast(lltype.Signed, ptr)
         # To hash pointers in dictionaries.  Assumes that h shows some
         # alignment (to 4, 8, maybe 16 bytes), so we use the following
         # formula to avoid the trailing bits being always 0.
@@ -128,26 +146,27 @@
             i = space.getindex_w(w_index, space.w_IndexError)
             ctype = self.ctype._check_subscript_index(self, i)
             w_o = self._do_getitem(ctype, i)
-        keepalive_until_here(self)
         return w_o
 
     def _do_getitem(self, ctype, i):
         ctitem = ctype.ctitem
-        return ctitem.convert_to_object(
-            rffi.ptradd(self._cdata, i * ctitem.size))
+        with self as ptr:
+            return ctitem.convert_to_object(
+                rffi.ptradd(ptr, i * ctitem.size))
 
     def setitem(self, w_index, w_value):
         space = self.space
         if space.isinstance_w(w_index, space.w_slice):
-            self._do_setslice(w_index, w_value)
+            with self as ptr:
+                self._do_setslice(w_index, w_value, ptr)
         else:
             i = space.getindex_w(w_index, space.w_IndexError)
             ctype = self.ctype._check_subscript_index(self, i)
             ctitem = ctype.ctitem
-            ctitem.convert_from_object(
-                rffi.ptradd(self._cdata, i * ctitem.size),
-                w_value)
-        keepalive_until_here(self)
+            with self as ptr:
+                ctitem.convert_from_object(
+                    rffi.ptradd(ptr, i * ctitem.size),
+                    w_value)
 
     def _do_getslicearg(self, w_slice):
         from pypy.module._cffi_backend.ctypeptr import W_CTypePointer
@@ -188,14 +207,15 @@
             ctarray = newtype.new_array_type(space, ctptr, space.w_None)
             ctptr.cache_array_type = ctarray
         #
-        p = rffi.ptradd(self._cdata, start * ctarray.ctitem.size)
-        return W_CDataSliced(space, p, ctarray, length)
+        ptr = self.unsafe_escaping_ptr()
+        ptr = rffi.ptradd(ptr, start * ctarray.ctitem.size)
+        return W_CDataSliced(space, ptr, ctarray, length)
 
-    def _do_setslice(self, w_slice, w_value):
+    def _do_setslice(self, w_slice, w_value, ptr):
         ctptr, start, length = self._do_getslicearg(w_slice)
         ctitem = ctptr.ctitem
         ctitemsize = ctitem.size
-        cdata = rffi.ptradd(self._cdata, start * ctitemsize)
+        target = rffi.ptradd(ptr, start * ctitemsize)
         #
         if isinstance(w_value, W_CData):
             from pypy.module._cffi_backend import ctypearray
@@ -204,9 +224,8 @@
                 ctv.ctitem is ctitem and
                 w_value.get_array_length() == length):
                 # fast path: copying from exactly the correct type
-                s = w_value._cdata
-                rffi.c_memcpy(cdata, s, ctitemsize * length)
-                keepalive_until_here(w_value)
+                with w_value as source:
+                    rffi.c_memcpy(target, source, ctitemsize * length)
                 return
         #
         # A fast path for [0:N] = "somestring".
@@ -221,7 +240,7 @@
                 raise oefmt(space.w_ValueError,
                             "need a string of length %d, got %d",
                             length, len(value))
-            copy_string_to_raw(llstr(value), cdata, 0, length)
+            copy_string_to_raw(llstr(value), target, 0, length)
             return
         #
         w_iter = space.iter(w_value)
@@ -233,8 +252,8 @@
                     raise
                 raise oefmt(space.w_ValueError,
                             "need %d values to unpack, got %d", length, i)
-            ctitem.convert_from_object(cdata, w_item)
-            cdata = rffi.ptradd(cdata, ctitemsize)
+            ctitem.convert_from_object(target, w_item)
+            target = rffi.ptradd(target, ctitemsize)
         try:
             space.next(w_iter)
         except OperationError, e:
@@ -247,7 +266,8 @@
     def _add_or_sub(self, w_other, sign):
         space = self.space
         i = sign * space.getindex_w(w_other, space.w_OverflowError)
-        return self.ctype.add(self._cdata, i)
+        ptr = self.unsafe_escaping_ptr()
+        return self.ctype.add(ptr, i)
 
     def add(self, w_other):
         return self._add_or_sub(w_other, +1)
@@ -268,9 +288,11 @@
                             self.ctype.name, ct.name)
             #
             itemsize = ct.ctitem.size
-            if itemsize <= 0: itemsize = 1
-            diff = (rffi.cast(lltype.Signed, self._cdata) -
-                    rffi.cast(lltype.Signed, w_other._cdata)) // itemsize
+            if itemsize <= 0:
+                itemsize = 1
+            with self as ptr1, w_other as ptr2:
+                diff = (rffi.cast(lltype.Signed, ptr1) -
+                        rffi.cast(lltype.Signed, ptr2)) // itemsize
             return space.wrap(diff)
         #
         return self._add_or_sub(w_other, -1)
@@ -279,17 +301,19 @@
         return self.ctype.getcfield(self.space.str_w(w_attr))
 
     def getattr(self, w_attr):
-        w_res = self.getcfield(w_attr).read(self._cdata)
-        keepalive_until_here(self)
+        cfield = self.getcfield(w_attr)
+        with self as ptr:
+            w_res = cfield.read(ptr)
         return w_res
 
     def setattr(self, w_attr, w_value):
-        self.getcfield(w_attr).write(self._cdata, w_value)
-        keepalive_until_here(self)
+        cfield = self.getcfield(w_attr)
+        with self as ptr:
+            cfield.write(ptr, w_value)
 
     def call(self, args_w):
-        w_result = self.ctype.call(self._cdata, args_w)
-        keepalive_until_here(self)
+        with self as ptr:
+            w_result = self.ctype.call(ptr, args_w)
         return w_result
 
     def iter(self):
@@ -311,21 +335,21 @@
 
     @specialize.argtype(1)
     def write_raw_signed_data(self, source):
-        misc.write_raw_signed_data(self._cdata, source, self.ctype.size)
-        keepalive_until_here(self)
+        with self as ptr:
+            misc.write_raw_signed_data(ptr, source, self.ctype.size)
 
     @specialize.argtype(1)
     def write_raw_unsigned_data(self, source):
-        misc.write_raw_unsigned_data(self._cdata, source, self.ctype.size)
-        keepalive_until_here(self)
+        with self as ptr:
+            misc.write_raw_unsigned_data(ptr, source, self.ctype.size)
 
     def write_raw_float_data(self, source):
-        misc.write_raw_float_data(self._cdata, source, self.ctype.size)
-        keepalive_until_here(self)
+        with self as ptr:
+            misc.write_raw_float_data(ptr, source, self.ctype.size)
 
     def convert_to_object(self):
-        w_obj = self.ctype.convert_to_object(self._cdata)
-        keepalive_until_here(self)
+        with self as ptr:
+            w_obj = self.ctype.convert_to_object(ptr)
         return w_obj
 
     def get_array_length(self):
@@ -353,7 +377,7 @@
 
     @rgc.must_be_light_finalizer
     def __del__(self):
-        lltype.free(self._cdata, flavor='raw')
+        lltype.free(self._ptr, flavor='raw')
 
 
 class W_CDataNewOwning(W_CDataMem):
diff --git a/pypy/module/_cffi_backend/cerrno.py b/pypy/module/_cffi_backend/cerrno.py
--- a/pypy/module/_cffi_backend/cerrno.py
+++ b/pypy/module/_cffi_backend/cerrno.py
@@ -13,18 +13,18 @@
 _errno_after  = rposix._errno_after
 
 def get_errno(space):
-    return space.wrap(rposix.get_saved_errno())
+    return space.wrap(rposix.get_saved_alterrno())
 
 @unwrap_spec(errno=int)
 def set_errno(space, errno):
-    rposix.set_saved_errno(errno)
+    rposix.set_saved_alterrno(errno)
 
 # ____________________________________________________________
 
 @unwrap_spec(code=int)
 def getwinerror(space, code=-1):
-    from rpython.rlib.rwin32 import GetLastError_saved, FormatError
+    from rpython.rlib.rwin32 import GetLastError_alt_saved, FormatError
     if code == -1:
-        code = GetLastError_saved()
+        code = GetLastError_alt_saved()
     message = FormatError(code)
     return space.newtuple([space.wrap(code), space.wrap(message)])
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -8,7 +8,6 @@
 from pypy.interpreter.typedef import TypeDef
 
 from rpython.rtyper.lltypesystem import rffi
-from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib.rarithmetic import ovfcheck
 
 from pypy.module._cffi_backend import cdataobj
@@ -49,8 +48,8 @@
             cdata = cdataobj.W_CDataNewOwning(space, datasize, self)
         #
         if not space.is_w(w_init, space.w_None):
-            self.convert_from_object(cdata._cdata, w_init)
-            keepalive_until_here(cdata)
+            with cdata as ptr:
+                self.convert_from_object(ptr, w_init)
         return cdata
 
     def _check_subscript_index(self, w_cdata, i):
@@ -119,8 +118,8 @@
         self.ctitem = ctitem
         self.cdata = cdata
         length = cdata.get_array_length()
-        self._next = cdata._cdata
-        self._stop = rffi.ptradd(cdata._cdata, length * ctitem.size)
+        self._next = cdata.unsafe_escaping_ptr()
+        self._stop = rffi.ptradd(self._next, length * ctitem.size)
 
     def iter_w(self):
         return self.space.wrap(self)
diff --git a/pypy/module/_cffi_backend/ctypeenum.py b/pypy/module/_cffi_backend/ctypeenum.py
--- a/pypy/module/_cffi_backend/ctypeenum.py
+++ b/pypy/module/_cffi_backend/ctypeenum.py
@@ -2,8 +2,6 @@
 Enums.
 """
 
-from rpython.rlib.objectmodel import keepalive_until_here
-
 from pypy.module._cffi_backend import misc
 from pypy.module._cffi_backend.ctypeprim import (W_CTypePrimitiveSigned,
     W_CTypePrimitiveUnsigned)
@@ -47,8 +45,8 @@
             return '%s: %s' % (value, s)
 
     def string(self, cdataobj, maxlen):
-        value = self._get_value(cdataobj._cdata)
-        keepalive_until_here(cdataobj)
+        with cdataobj as ptr:
+            value = self._get_value(ptr)
         try:
             s = self.enumvalues2erators[value]
         except KeyError:
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -177,8 +177,8 @@
         raise oefmt(space.w_AttributeError,
                     "cdata '%s' has no attribute '%s'", self.name, attr)
 
-    def copy_and_convert_to_object(self, cdata):
-        return self.convert_to_object(cdata)
+    def copy_and_convert_to_object(self, source):
+        return self.convert_to_object(source)
 
     # __________ app-level attributes __________
     def dir(self):
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -5,7 +5,6 @@
 import sys
 
 from rpython.rlib.rarithmetic import r_uint, r_ulonglong, intmask
-from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib import jit
 from rpython.rtyper.lltypesystem import lltype, rffi
 
@@ -53,7 +52,8 @@
         space = self.space
         if (isinstance(w_ob, cdataobj.W_CData) and
                isinstance(w_ob.ctype, ctypeptr.W_CTypePtrOrArray)):
-            value = rffi.cast(lltype.Signed, w_ob._cdata)
+            ptr = w_ob.unsafe_escaping_ptr()
+            value = rffi.cast(lltype.Signed, ptr)
             value = self._cast_result(value)
         elif space.isinstance_w(w_ob, space.w_str):
             value = self.cast_str(w_ob)
@@ -81,8 +81,8 @@
 
     def string(self, cdataobj, maxlen):
         if self.size == 1:
-            s = cdataobj._cdata[0]
-            keepalive_until_here(cdataobj)
+            with cdataobj as ptr:
+                s = ptr[0]
             return self.space.wrap(s)
         return W_CType.string(self, cdataobj, maxlen)
 
@@ -116,7 +116,8 @@
                 return s[0]
         if (isinstance(w_ob, cdataobj.W_CData) and
                isinstance(w_ob.ctype, W_CTypePrimitiveChar)):
-            return w_ob._cdata[0]
+            with w_ob as ptr:
+                return ptr[0]
         raise self._convert_error("string of length 1", w_ob)
 
     def convert_from_object(self, cdata, w_ob):
@@ -137,8 +138,8 @@
         return self.space.wrap(s)
 
     def string(self, cdataobj, maxlen):
-        w_res = self.convert_to_object(cdataobj._cdata)
-        keepalive_until_here(cdataobj)
+        with cdataobj as ptr:
+            w_res = self.convert_to_object(ptr)
         return w_res
 
     def _convert_to_unichar(self, w_ob):
@@ -149,7 +150,8 @@
                 return s[0]
         if (isinstance(w_ob, cdataobj.W_CData) and
                isinstance(w_ob.ctype, W_CTypePrimitiveUniChar)):
-            return rffi.cast(rffi.CWCHARP, w_ob._cdata)[0]
+            with w_ob as ptr:
+                return rffi.cast(rffi.CWCHARP, ptr)[0]
         raise self._convert_error("unicode string of length 1", w_ob)
 
     def convert_from_object(self, cdata, w_ob):
@@ -219,13 +221,15 @@
         if self.size == rffi.sizeof(rffi.LONG):
             from rpython.rlib.rrawarray import populate_list_from_raw_array
             res = []
-            buf = rffi.cast(rffi.LONGP, w_cdata._cdata)
             length = w_cdata.get_array_length()
-            populate_list_from_raw_array(res, buf, length)
+            with w_cdata as ptr:
+                buf = rffi.cast(rffi.LONGP, ptr)
+                populate_list_from_raw_array(res, buf, length)
             return res
         elif self.value_smaller_than_long:
             res = [0] * w_cdata.get_array_length()
-            misc.unpack_list_from_raw_array(res, w_cdata._cdata, self.size)
+            with w_cdata as ptr:
+                misc.unpack_list_from_raw_array(res, ptr, self.size)
             return res
         return None
 
@@ -308,8 +312,8 @@
     def unpack_list_of_int_items(self, w_cdata):
         if self.value_fits_long:
             res = [0] * w_cdata.get_array_length()
-            misc.unpack_unsigned_list_from_raw_array(res, w_cdata._cdata,
-                                                     self.size)
+            with w_cdata as ptr:
+                misc.unpack_unsigned_list_from_raw_array(res, ptr, self.size)
             return res
         return None
 
@@ -363,8 +367,8 @@
         if not isinstance(self, W_CTypePrimitiveLongDouble):
             w_cdata.write_raw_float_data(value)
         else:
-            self._to_longdouble_and_write(value, w_cdata._cdata)
-            keepalive_until_here(w_cdata)
+            with w_cdata as ptr:
+                self._to_longdouble_and_write(value, ptr)
         return w_cdata
 
     def cast_to_int(self, cdata):
@@ -387,13 +391,15 @@
         if self.size == rffi.sizeof(rffi.DOUBLE):
             from rpython.rlib.rrawarray import populate_list_from_raw_array
             res = []
-            buf = rffi.cast(rffi.DOUBLEP, w_cdata._cdata)
             length = w_cdata.get_array_length()
-            populate_list_from_raw_array(res, buf, length)
+            with w_cdata as ptr:
+                buf = rffi.cast(rffi.DOUBLEP, ptr)
+                populate_list_from_raw_array(res, buf, length)
             return res
         elif self.size == rffi.sizeof(rffi.FLOAT):
             res = [0.0] * w_cdata.get_array_length()
-            misc.unpack_cfloat_list_from_raw_array(res, w_cdata._cdata)
+            with w_cdata as ptr:
+                misc.unpack_cfloat_list_from_raw_array(res, ptr)
             return res
         return None
 
@@ -423,8 +429,8 @@
     def cast(self, w_ob):
         if (isinstance(w_ob, cdataobj.W_CData) and
                 isinstance(w_ob.ctype, W_CTypePrimitiveLongDouble)):
-            w_cdata = self.convert_to_object(w_ob._cdata)
-            keepalive_until_here(w_ob)
+            with w_ob as ptr:
+                w_cdata = self.convert_to_object(ptr)
             return w_cdata
         else:
             return W_CTypePrimitiveFloat.cast(self, w_ob)
@@ -451,16 +457,16 @@
 
     def convert_to_object(self, cdata):
         w_cdata = cdataobj.W_CDataMem(self.space, self.size, self)
-        self._copy_longdouble(cdata, w_cdata._cdata)
-        keepalive_until_here(w_cdata)
+        with w_cdata as ptr:
+            self._copy_longdouble(cdata, ptr)
         return w_cdata
 
     def convert_from_object(self, cdata, w_ob):
         space = self.space
         if (isinstance(w_ob, cdataobj.W_CData) and
                 isinstance(w_ob.ctype, W_CTypePrimitiveLongDouble)):
-            self._copy_longdouble(w_ob._cdata, cdata)
-            keepalive_until_here(w_ob)
+            with w_ob as ptr:
+                self._copy_longdouble(ptr, cdata)
         else:
             value = space.float_w(space.float(w_ob))
             self._to_longdouble_and_write(value, cdata)
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -3,7 +3,6 @@
 """
 
 from rpython.rlib import rposix
-from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rtyper.annlowlevel import llstr, llunicode
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -49,7 +48,7 @@
         space = self.space
         if (isinstance(w_ob, cdataobj.W_CData) and
                 isinstance(w_ob.ctype, W_CTypePtrOrArray)):
-            value = w_ob._cdata
+            value = w_ob.unsafe_escaping_ptr()
         else:
             value = misc.as_unsigned_long(space, w_ob, strict=False)
             value = rffi.cast(rffi.CCHARP, value)
@@ -108,34 +107,33 @@
     def string(self, cdataobj, maxlen):
         space = self.space
         if isinstance(self.ctitem, ctypeprim.W_CTypePrimitive):
-            cdata = cdataobj._cdata
-            if not cdata:
-                raise oefmt(space.w_RuntimeError, "cannot use string() on %s",
-                            space.str_w(cdataobj.repr()))
-            #
-            from pypy.module._cffi_backend import ctypearray
-            length = maxlen
-            if length < 0 and isinstance(self, ctypearray.W_CTypeArray):
-                length = cdataobj.get_array_length()
-            #
-            # pointer to a primitive type of size 1: builds and returns a str
-            if self.ctitem.size == rffi.sizeof(lltype.Char):
-                if length < 0:
-                    s = rffi.charp2str(cdata)
-                else:
-                    s = rffi.charp2strn(cdata, length)
-                keepalive_until_here(cdataobj)
-                return space.wrap(s)
-            #
-            # pointer to a wchar_t: builds and returns a unicode
-            if self.is_unichar_ptr_or_array():
-                cdata = rffi.cast(rffi.CWCHARP, cdata)
-                if length < 0:
-                    u = rffi.wcharp2unicode(cdata)
-                else:
-                    u = rffi.wcharp2unicoden(cdata, length)
-                keepalive_until_here(cdataobj)
-                return space.wrap(u)
+            with cdataobj as ptr:
+                if not ptr:
+                    raise oefmt(space.w_RuntimeError,
+                                "cannot use string() on %s",
+                                space.str_w(cdataobj.repr()))
+                #
+                from pypy.module._cffi_backend import ctypearray
+                length = maxlen
+                if length < 0 and isinstance(self, ctypearray.W_CTypeArray):
+                    length = cdataobj.get_array_length()
+                #
+                # pointer to a primitive type of size 1: builds and returns a str
+                if self.ctitem.size == rffi.sizeof(lltype.Char):
+                    if length < 0:
+                        s = rffi.charp2str(ptr)
+                    else:
+                        s = rffi.charp2strn(ptr, length)
+                    return space.wrap(s)
+                #
+                # pointer to a wchar_t: builds and returns a unicode
+                if self.is_unichar_ptr_or_array():
+                    cdata = rffi.cast(rffi.CWCHARP, ptr)
+                    if length < 0:
+                        u = rffi.wcharp2unicode(cdata)
+                    else:
+                        u = rffi.wcharp2unicoden(cdata, length)
+                    return space.wrap(u)
         #
         return W_CType.string(self, cdataobj, maxlen)
 
@@ -162,7 +160,7 @@
             if not (self.can_cast_anything or other.can_cast_anything):
                 raise self._convert_error("compatible pointer", w_ob)
 
-        rffi.cast(rffi.CCHARPP, cdata)[0] = w_ob._cdata
+        rffi.cast(rffi.CCHARPP, cdata)[0] = w_ob.unsafe_escaping_ptr()
 
     def _alignof(self):
         from pypy.module._cffi_backend import newtype
@@ -206,8 +204,8 @@
                     lltype.nullptr(rffi.CCHARP.TO), w_init, datasize)
             #
             cdatastruct = cdataobj.W_CDataNewOwning(space, datasize, ctitem)
-            cdata = cdataobj.W_CDataPtrToStructOrUnion(space,
-                                                       cdatastruct._cdata,
+            ptr = cdatastruct.unsafe_escaping_ptr()
+            cdata = cdataobj.W_CDataPtrToStructOrUnion(space, ptr,
                                                        self, cdatastruct)
         else:
             if self.is_char_or_unichar_ptr_or_array():
@@ -215,8 +213,8 @@
             cdata = cdataobj.W_CDataNewOwning(space, datasize, self)
         #
         if not space.is_w(w_init, space.w_None):
-            ctitem.convert_from_object(cdata._cdata, w_init)
-            keepalive_until_here(cdata)
+            with cdata as ptr:
+                ctitem.convert_from_object(ptr, w_init)
         return cdata
 
     def _check_subscript_index(self, w_cdata, i):
@@ -332,8 +330,9 @@
         ctype2 = cdata.ctype
         if (isinstance(ctype2, W_CTypeStructOrUnion) or
                 isinstance(ctype2, W_CTypePtrOrArray)):
-            ptrdata = rffi.ptradd(cdata._cdata, offset)
-            return cdataobj.W_CData(space, ptrdata, self)
+            ptr = cdata.unsafe_escaping_ptr()
+            ptr = rffi.ptradd(ptr, offset)
+            return cdataobj.W_CData(space, ptr, self)
         else:
             raise OperationError(space.w_TypeError,
                     space.wrap("expected a cdata struct/union/array/pointer"
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -7,7 +7,6 @@
 from pypy.interpreter.typedef import TypeDef, interp_attrproperty
 
 from rpython.rlib import jit
-from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib.rarithmetic import r_uint, r_ulonglong, r_longlong, intmask
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -57,12 +56,12 @@
         self.check_complete()
         return cdataobj.W_CData(space, cdata, self)
 
-    def copy_and_convert_to_object(self, cdata):
+    def copy_and_convert_to_object(self, source):
         space = self.space
         self.check_complete()
         ob = cdataobj.W_CDataNewOwning(space, self.size, self)
-        misc._raw_memcopy(cdata, ob._cdata, self.size)
-        keepalive_until_here(ob)
+        with ob as target:
+            misc._raw_memcopy(source, target, self.size)
         return ob
 
     def typeoffsetof_field(self, fieldname, following):
@@ -80,8 +79,8 @@
     def _copy_from_same(self, cdata, w_ob):
         if isinstance(w_ob, cdataobj.W_CData):
             if w_ob.ctype is self and self.size >= 0:
-                misc._raw_memcopy(w_ob._cdata, cdata, self.size)
-                keepalive_until_here(w_ob)
+                with w_ob as ptr:
+                    misc._raw_memcopy(ptr, cdata, self.size)
                 return True
         return False
 
diff --git a/pypy/module/_cffi_backend/ctypevoid.py b/pypy/module/_cffi_backend/ctypevoid.py
--- a/pypy/module/_cffi_backend/ctypevoid.py
+++ b/pypy/module/_cffi_backend/ctypevoid.py
@@ -13,5 +13,5 @@
     def __init__(self, space):
         W_CType.__init__(self, space, -1, "void", len("void"))
 
-    def copy_and_convert_to_object(self, cdata):
+    def copy_and_convert_to_object(self, source):
         return self.space.w_None
diff --git a/pypy/module/_cffi_backend/handle.py b/pypy/module/_cffi_backend/handle.py
--- a/pypy/module/_cffi_backend/handle.py
+++ b/pypy/module/_cffi_backend/handle.py
@@ -34,8 +34,9 @@
         raise oefmt(space.w_TypeError,
                     "expected a 'cdata' object with a 'void *' out of "
                     "new_handle(), got '%s'", ctype.name)
-    index = rffi.cast(lltype.Signed, w_cdata._cdata)
-    original_cdataobj = get(space).fetch_handle(index - 1)
+    with w_cdata as ptr:
+        index = rffi.cast(lltype.Signed, ptr)
+        original_cdataobj = get(space).fetch_handle(index - 1)
     #
     if isinstance(original_cdataobj, cdataobj.W_CDataHandle):
         return original_cdataobj.w_keepalive
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -3,7 +3,7 @@
 from pypy.interpreter.error import OperationError
 
 from rpython.rlib import jit
-from rpython.rlib.objectmodel import keepalive_until_here, specialize
+from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rarithmetic import r_uint, r_ulonglong
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
@@ -272,11 +272,11 @@
     from pypy.module._cffi_backend.ctypeprim import W_CTypePrimitiveLongDouble
     is_cdata = isinstance(w_ob, W_CData)
     if is_cdata and isinstance(w_ob.ctype, W_CTypePrimitiveFloat):
-        if isinstance(w_ob.ctype, W_CTypePrimitiveLongDouble):
-            result = is_nonnull_longdouble(w_ob._cdata)
-        else:
-            result = is_nonnull_float(w_ob._cdata, w_ob.ctype.size)
-        keepalive_until_here(w_ob)
+        with w_ob as ptr:
+            if isinstance(w_ob.ctype, W_CTypePrimitiveLongDouble):
+                result = is_nonnull_longdouble(ptr)
+            else:
+                result = is_nonnull_float(ptr, w_ob.ctype.size)
         return result
     #
     if not is_cdata and space.lookup(w_ob, '__float__') is not None:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -2716,6 +2716,14 @@
     assert data == b"Xhello\n"
     posix.close(fdr)
 
+def test_errno_saved():
+    set_errno(42)
+    # a random function that will reset errno to 0 (at least on non-windows)
+    import os; os.stat('.')
+    #
+    res = get_errno()
+    assert res == 42
+
 def test_GetLastError():
     if sys.platform != "win32":
         py.test.skip("GetLastError(): only for Windows")
@@ -3241,4 +3249,4 @@
 
 def test_version():
     # this test is here mostly for PyPy
-    assert __version__ == "0.8.6+"
+    assert __version__ == "0.9.1"
diff --git a/pypy/module/_cffi_backend/test/test_file.py b/pypy/module/_cffi_backend/test/test_file.py
--- a/pypy/module/_cffi_backend/test/test_file.py
+++ b/pypy/module/_cffi_backend/test/test_file.py
@@ -15,3 +15,15 @@
             "Update test/_backend_test_c.py by copying it from "
             "https://bitbucket.org/cffi/cffi/raw/default/c/test_c.py "
             "and killing the import lines at the start")
+
+def test_egginfo_version():
+    from pypy.module._cffi_backend import VERSION
+    line = "Version: %s\n" % VERSION
+    eggfile = py.path.local(__file__).join('..', '..', '..', '..', '..',
+                                           'lib_pypy', 'cffi.egg-info')
+    assert line in eggfile.readlines()
+
+def test_app_version():
+    from pypy.module import _cffi_backend
+    from lib_pypy import cffi
+    assert _cffi_backend.VERSION == cffi.__version__
diff --git a/pypy/module/_random/interp_random.py b/pypy/module/_random/interp_random.py
--- a/pypy/module/_random/interp_random.py
+++ b/pypy/module/_random/interp_random.py
@@ -4,7 +4,7 @@
 from pypy.interpreter.typedef import TypeDef
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.baseobjspace import W_Root
-from rpython.rlib.rarithmetic import r_uint, intmask
+from rpython.rlib.rarithmetic import r_uint, intmask, widen
 from rpython.rlib import rbigint, rrandom, rstring
 
 
@@ -54,8 +54,8 @@
     def getstate(self, space):
         state = [None] * (rrandom.N + 1)
         for i in range(rrandom.N):
-            state[i] = space.newint(intmask(self._rnd.state[i]))
-        state[rrandom.N] = space.newint(self._rnd.index)
+            state[i] = space.wrap(widen(self._rnd.state[i]))
+        state[rrandom.N] = space.newlong(self._rnd.index)
         return space.newtuple(state)
 
     def setstate(self, space, w_state):
diff --git a/pypy/module/_random/test/test_random.py b/pypy/module/_random/test/test_random.py
--- a/pypy/module/_random/test/test_random.py
+++ b/pypy/module/_random/test/test_random.py
@@ -41,6 +41,17 @@
         # does not crash
         rnd1.setstate((-1, ) * 624 + (0, ))
 
+    def test_state_repr(self):
+        # since app-level jumpahead salts with repr(state),
+        # it is important the repr is consistent with cpython
+        import _random
+        rnd = _random.Random()
+        rnd.seed(1234)
+        state = rnd.getstate()
+        s = repr(state)
+        assert len(s) == 7956
+        assert s.count('L') == 625
+
     def test_seed(self):
         import _random, sys
         rnd = _random.Random()
@@ -102,3 +113,10 @@
                 self.x = x
         r = R(x=15)
         assert r.x == 15
+
+    def test_exact_result(self):
+        # this passes on CPython 2.7.9 on Linux 32 and Linux 64
+        import _random
+        rnd = _random.Random(-3**31)
+        x = rnd.random()
+        assert x == 0.8181851342382107
diff --git a/pypy/module/_rawffi/array.py b/pypy/module/_rawffi/array.py
--- a/pypy/module/_rawffi/array.py
+++ b/pypy/module/_rawffi/array.py
@@ -15,7 +15,7 @@
 from pypy.module._rawffi.interp_rawffi import unpack_shape_with_length
 from pypy.module._rawffi.interp_rawffi import read_ptr, write_ptr
 from rpython.rlib.rarithmetic import r_uint
-from rpython.rlib import rgc
+from rpython.rlib import rgc, clibffi
 
 
 class W_Array(W_DataShape):
@@ -84,14 +84,11 @@
 
 class W_ArrayInstance(W_DataInstance):
     def __init__(self, space, shape, length, address=r_uint(0)):
-        # Workaround for a strange behavior of libffi: make sure that
-        # we always have at least 8 bytes.  For W_ArrayInstances that are
-        # used as the result value of a function call, ffi_call() writes
-        # 8 bytes into it even if the function's result type asks for less.
-        # This strange behavior is documented.
         memsize = shape.size * length
-        if memsize < 8:
-            memsize = 8
+        # For W_ArrayInstances that are used as the result value of a
+        # function call, ffi_call() writes 8 bytes into it even if the
+        # function's result type asks for less.
+        memsize = clibffi.adjust_return_size(memsize)
         W_DataInstance.__init__(self, space, memsize, address)
         self.length = length
         self.shape = shape
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -495,6 +495,7 @@
         try:
             if self.resshape is not None:
                 result = self.resshape.allocate(space, 1, autofree=True)
+                # adjust_return_size() was used here on result.ll_buffer
                 self.ptr.call(args_ll, result.ll_buffer)
                 return space.wrap(result)
             else:
@@ -608,19 +609,19 @@
     return space.wrap(W_CDLL(space, name, cdll))
 
 def get_errno(space):
-    return space.wrap(rposix.get_saved_errno())
+    return space.wrap(rposix.get_saved_alterrno())
 
 def set_errno(space, w_errno):
-    rposix.set_saved_errno(space.int_w(w_errno))
+    rposix.set_saved_alterrno(space.int_w(w_errno))
 
 if sys.platform == 'win32':
     # see also
     # https://bitbucket.org/pypy/pypy/issue/1944/ctypes-on-windows-getlasterror
     def get_last_error(space):
-        return space.wrap(rwin32.GetLastError_saved())
+        return space.wrap(rwin32.GetLastError_alt_saved())
     @unwrap_spec(error=int)
     def set_last_error(space, error):
-        rwin32.SetLastError_saved(error)
+        rwin32.SetLastError_alt_saved(error)
 else:
     # always have at least a dummy version of these functions
     # (https://bugs.pypy.org/issue1242)
diff --git a/pypy/module/_ssl/__init__.py b/pypy/module/_ssl/__init__.py
--- a/pypy/module/_ssl/__init__.py
+++ b/pypy/module/_ssl/__init__.py
@@ -51,6 +51,11 @@
 
         super(Module, cls).buildloaders()
 
+    def setup_after_space_initialization(self):
+        """NOT_RPYTHON"""
+        from pypy.module._ssl.interp_ssl import PWINFO_STORAGE
+        PWINFO_STORAGE.clear()
+
     def startup(self, space):
         from rpython.rlib.ropenssl import init_ssl
         init_ssl()
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -1,6 +1,8 @@
 from rpython.rlib import rpoll, rsocket, rthread, rweakref
 from rpython.rlib.rarithmetic import intmask, widen, r_uint
 from rpython.rlib.ropenssl import *
+from pypy.module._socket import interp_socket
+from rpython.rlib._rsocket_rffi import MAX_FD_SIZE
 from rpython.rlib.rposix import get_saved_errno
 from rpython.rlib.rweakref import RWeakValueDictionary
 from rpython.rlib.objectmodel import specialize, compute_unique_id
@@ -12,7 +14,6 @@
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.module._ssl.ssl_data import (
     LIBRARY_CODES_TO_NAMES, ERROR_CODES_TO_NAMES)
-from pypy.module._socket import interp_socket
 
 
 # user defined constants
@@ -79,7 +80,8 @@
 constants["OP_CIPHER_SERVER_PREFERENCE"] = SSL_OP_CIPHER_SERVER_PREFERENCE
 constants["OP_SINGLE_DH_USE"] = SSL_OP_SINGLE_DH_USE
 constants["OP_SINGLE_ECDH_USE"] = SSL_OP_SINGLE_ECDH_USE
-constants["OP_NO_COMPRESSION"] = SSL_OP_NO_COMPRESSION
+if SSL_OP_NO_COMPRESSION is not None:
+    constants["OP_NO_COMPRESSION"] = SSL_OP_NO_COMPRESSION
 
 constants["OPENSSL_VERSION_NUMBER"] = OPENSSL_VERSION_NUMBER
 ver = OPENSSL_VERSION_NUMBER
@@ -886,9 +888,13 @@
         libssl_AUTHORITY_INFO_ACCESS_free(info)
 
 def _get_crl_dp(space, certificate):
-    # Calls x509v3_cache_extensions and sets up crldp
-    libssl_X509_check_ca(certificate)
-    dps = certificate[0].c_crldp
+    if OPENSSL_VERSION_NUMBER >= 0x10001000:
+        # Calls x509v3_cache_extensions and sets up crldp
+        libssl_X509_check_ca(certificate)
+        dps = certificate[0].c_crldp
+    else:
+        dps = rffi.cast(stack_st_DIST_POINT, libssl_X509_get_ext_d2i(
+            certificate, NID_crl_distribution_points, None, None))
     if not dps:
         return None
 
diff --git a/pypy/module/_ssl/ssl_data.py b/pypy/module/_ssl/ssl_data.py
--- a/pypy/module/_ssl/ssl_data.py
+++ b/pypy/module/_ssl/ssl_data.py
@@ -369,7 +369,8 @@
     LIBRARY_CODES_TO_NAMES[cconfig[code]] = code
 ERROR_CODES_TO_NAMES = {}
 for lib, code in error_codes:
-    ERROR_CODES_TO_NAMES[cconfig[lib], cconfig[code]] = code
+    if cconfig[code] is not None:
+        ERROR_CODES_TO_NAMES[cconfig[lib], cconfig[code]] = code
 
 ALERT_DESCRIPTION_CODES = {}
 for name in AD_NAMES:
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -236,6 +236,9 @@
 
     def test_npn_protocol(self):
         import socket, _ssl, gc
+        if not _ssl.HAS_NPN:
+            skip("NPN requires OpenSSL 1.0.1 or greater")
+
         ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
         ctx._set_npn_protocols(b'\x08http/1.1\x06spdy/2')
         ss = ctx._wrap_socket(self.s._sock, True,
@@ -307,12 +310,13 @@
             os.path.dirname(__file__), 'dh512.pem'))
 
     def test_load_cert_chain(self):
-        import _ssl
+        import _ssl, errno
         ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
         ctx.load_cert_chain(self.keycert)
         ctx.load_cert_chain(self.cert, self.key)
-        raises(IOError, ctx.load_cert_chain, "inexistent.pem")
-        raises(_ssl.SSLError, ctx.load_cert_chain, self.badcert)
+        exc = raises(IOError, ctx.load_cert_chain, "inexistent.pem")
+        assert exc.value.errno == errno.ENOENT
+        exc = raises(_ssl.SSLError, ctx.load_cert_chain, self.badcert)
         raises(_ssl.SSLError, ctx.load_cert_chain, self.emptycert)
         # Password protected key and cert
         raises(_ssl.SSLError, ctx.load_cert_chain, self.cert_protected,
@@ -360,12 +364,14 @@
         assert ctx.cert_store_stats() == {'x509_ca': 0, 'crl': 0, 'x509': 1}
 
     def test_load_dh_params(self):
-        import _ssl
+        import _ssl, errno
         ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
         ctx.load_dh_params(self.dh512)
         raises(TypeError, ctx.load_dh_params)
         raises(TypeError, ctx.load_dh_params, None)
         raises(_ssl.SSLError, ctx.load_dh_params, self.keycert)
+        exc = raises(IOError, ctx.load_dh_params, "inexistent.pem")
+        assert exc.value.errno == errno.ENOENT
 
     def test_set_ecdh_curve(self):
         import _ssl
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -1,4 +1,4 @@
-from rpython.rlib import jit
+from rpython.rlib import jit, rgc
 from rpython.rlib.buffer import Buffer
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib.rarithmetic import ovfcheck, widen
@@ -698,11 +698,10 @@
                                          self.space.wrap(msg))
             return result
 
+        @rgc.must_be_light_finalizer
         def __del__(self):
-            # note that we don't call clear_all_weakrefs here because
-            # an array with freed buffer is ok to see - it's just empty with 0
-            # length
-            self.setlen(0)
+            if self.buffer:
+                lltype.free(self.buffer, flavor='raw')
 
         def setlen(self, size, zero=False, overallocate=True):
             if size > 0:
diff --git a/pypy/module/cppyy/capi/loadable_capi.py b/pypy/module/cppyy/capi/loadable_capi.py
--- a/pypy/module/cppyy/capi/loadable_capi.py
+++ b/pypy/module/cppyy/capi/loadable_capi.py
@@ -259,7 +259,8 @@
         if not objectmodel.we_are_translated():
             leakfinder.remember_free(c_call.ctype.cif_descr._obj0)
         state.capi_calls[name] = c_call
-    return c_call.ctype.rcall(c_call._cdata, args)
+    with c_call as ptr:
+        return c_call.ctype.rcall(ptr, args)
 
 def _cdata_to_cobject(space, w_cdata):
     return rffi.cast(C_OBJECT, space.uint_w(w_cdata))
@@ -271,8 +272,9 @@
     return rffi.cast(rffi.LONG, space.int_w(w_cdata))
 
 def _cdata_to_ptr(space, w_cdata): # TODO: this is both a hack and dreadfully slow
-    return rffi.cast(rffi.VOIDP,
-        space.interp_w(cdataobj.W_CData, w_cdata, can_be_None=False)._cdata)
+    w_cdata = space.interp_w(cdataobj.W_CData, w_cdata, can_be_None=False)
+    ptr = w_cdata.unsafe_escaping_ptr()
+    return rffi.cast(rffi.VOIDP, ptr)
 
 def c_load_dictionary(name):
     return libffi.CDLL(name)
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,7 +29,7 @@
 #define PY_VERSION		"2.7.9"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "2.6.0"
+#define PYPY_VERSION "2.6.0-alpha0"
 
 /* Subversion Revision number of this file (not of the repository).
  * Empty since Mercurial migration. */
diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py
--- a/pypy/module/micronumpy/base.py
+++ b/pypy/module/micronumpy/base.py
@@ -46,7 +46,7 @@
     @staticmethod
     def from_shape_and_storage(space, shape, storage, dtype, storage_bytes=-1,
                                order='C', owning=False, w_subtype=None,
-                               w_base=None, writable=True, strides=None):
+                               w_base=None, writable=True, strides=None, start=0):
         from pypy.module.micronumpy import concrete
         from pypy.module.micronumpy.strides import (calc_strides,
                                                     calc_backstrides)
@@ -75,8 +75,9 @@
                 raise OperationError(space.w_ValueError,
                         space.wrap("Cannot have owning=True when specifying a buffer"))
             if writable:
-                impl = concrete.ConcreteArrayWithBase(shape, dtype, order, strides,
-                                                      backstrides, storage, w_base)
+                impl = concrete.ConcreteArrayWithBase(shape, dtype, order,
+                                    strides, backstrides, storage, w_base,
+                                    start=start)
             else:
                 impl = concrete.ConcreteNonWritableArrayWithBase(shape, dtype, order,
                                                                  strides, backstrides,
@@ -128,6 +129,9 @@
     def get_order(self):
         return self.implementation.order
 
+    def get_start(self):
+        return self.implementation.start
+
     def ndims(self):
         return len(self.get_shape())
     ndims._always_inline_ = True
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -12,6 +12,7 @@
 from pypy.module.micronumpy.strides import (Chunk, Chunks, NewAxisChunk,
     RecordChunk, calc_strides, calc_new_strides, shape_agreement,
     calculate_broadcast_strides, calc_backstrides)
+from rpython.rlib.objectmodel import keepalive_until_here
 
 
 class BaseConcreteArray(object):
@@ -312,12 +313,15 @@
         l_w = [w_res.descr_getitem(space, space.wrap(d)) for d in range(nd)]
         return space.newtuple(l_w)
 
-    def get_storage_as_int(self, space):
-        return rffi.cast(lltype.Signed, self.storage) + self.start
-
-    def get_storage(self):
+    ##def get_storage(self):
+    ##    return self.storage
+    ## use a safer context manager
+    def __enter__(self):
         return self.storage
 
+    def __exit__(self, typ, value, traceback):
+        keepalive_until_here(self)
+        
     def get_buffer(self, space, readonly):
         return ArrayBuffer(self, readonly)
 
@@ -331,7 +335,7 @@
 
 
 class ConcreteArrayNotOwning(BaseConcreteArray):
-    def __init__(self, shape, dtype, order, strides, backstrides, storage):
+    def __init__(self, shape, dtype, order, strides, backstrides, storage, start=0):
         make_sure_not_resized(shape)
         make_sure_not_resized(strides)
         make_sure_not_resized(backstrides)
@@ -342,6 +346,7 @@
         self.strides = strides
         self.backstrides = backstrides
         self.storage = storage
+        self.start = start
 
     def fill(self, space, box):
         self.dtype.itemtype.fill(self.storage, self.dtype.elsize,
@@ -350,7 +355,7 @@
     def set_shape(self, space, orig_array, new_shape):
         strides, backstrides = calc_strides(new_shape, self.dtype,
                                                     self.order)
-        return SliceArray(0, strides, backstrides, new_shape, self,
+        return SliceArray(self.start, strides, backstrides, new_shape, self,
                           orig_array)
 
     def set_dtype(self, space, dtype):
@@ -384,9 +389,10 @@
 
 
 class ConcreteArrayWithBase(ConcreteArrayNotOwning):
-    def __init__(self, shape, dtype, order, strides, backstrides, storage, orig_base):
+    def __init__(self, shape, dtype, order, strides, backstrides, storage,
+                 orig_base, start=0):
         ConcreteArrayNotOwning.__init__(self, shape, dtype, order,
-                                        strides, backstrides, storage)
+                                        strides, backstrides, storage, start)
         self.orig_base = orig_base
 
     def base(self):
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -99,10 +99,12 @@
             for i in range(w_object.get_size()):
                 elems_w[i] = w_object.implementation.getitem(i * elsize)
         else:
-            sz = support.product(w_object.get_shape()) * dtype.elsize
-            return W_NDimArray.from_shape_and_storage(space,
-                w_object.get_shape(),w_object.implementation.storage,
-                dtype, storage_bytes=sz, w_base=w_object)
+            imp = w_object.implementation
+            with imp as storage:
+                sz = support.product(w_object.get_shape()) * dtype.elsize
+                return W_NDimArray.from_shape_and_storage(space,
+                    w_object.get_shape(), storage, dtype, storage_bytes=sz, 
+                    w_base=w_object, start=imp.start)
     else:
         # not an array
         shape, elems_w = strides.find_shape_and_elems(space, w_object, dtype)
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -604,14 +604,15 @@
     iter, state = arr.create_iter()
     w_res_str = W_NDimArray.from_shape(space, [1], arr.get_dtype(), order='C')
     itemsize = arr.get_dtype().elsize
-    res_str_casted = rffi.cast(rffi.CArrayPtr(lltype.Char),
-                               w_res_str.implementation.get_storage_as_int(space))
-    while not iter.done(state):
-        w_res_str.implementation.setitem(0, iter.getitem(state))
-        for i in range(itemsize):
-            builder.append(res_str_casted[i])
-        state = iter.next(state)
-    return builder.build()
+    with w_res_str.implementation as storage:
+        res_str_casted = rffi.cast(rffi.CArrayPtr(lltype.Char),
+                               support.get_storage_as_int(storage))
+        while not iter.done(state):
+            w_res_str.implementation.setitem(0, iter.getitem(state))
+            for i in range(itemsize):
+                builder.append(res_str_casted[i])
+            state = iter.next(state)
+        return builder.build()
 
 getitem_int_driver = jit.JitDriver(name = 'numpy_getitem_int',
                                    greens = ['shapelen', 'indexlen',
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -532,20 +532,25 @@
             self.get_dtype(), storage_bytes=sz, w_base=self)
 
     def descr_array_iface(self, space):
-        addr = self.implementation.get_storage_as_int(space)
-        # will explode if it can't
-        w_d = space.newdict()
-        space.setitem_str(w_d, 'data',
-                          space.newtuple([space.wrap(addr), space.w_False]))
-        space.setitem_str(w_d, 'shape', self.descr_get_shape(space))
-        space.setitem_str(w_d, 'typestr', self.get_dtype().descr_get_str(space))
-        if self.implementation.order == 'C':
-            # Array is contiguous, no strides in the interface.
-            strides = space.w_None
-        else:
-            strides = self.descr_get_strides(space)
-        space.setitem_str(w_d, 'strides', strides)
-        return w_d
+        '''
+        Note: arr.__array__.data[0] is a pointer so arr must be kept alive
+              while it is in use
+        '''
+        with self.implementation as storage:
+            addr = support.get_storage_as_int(storage, self.get_start())
+            # will explode if it can't
+            w_d = space.newdict()
+            space.setitem_str(w_d, 'data',
+                              space.newtuple([space.wrap(addr), space.w_False]))
+            space.setitem_str(w_d, 'shape', self.descr_get_shape(space))
+            space.setitem_str(w_d, 'typestr', self.get_dtype().descr_get_str(space))
+            if self.implementation.order == 'C':
+                # Array is contiguous, no strides in the interface.
+                strides = space.w_None
+            else:
+                strides = self.descr_get_strides(space)
+            space.setitem_str(w_d, 'strides', strides)
+            return w_d
 
     w_pypy_data = None
 
@@ -1165,7 +1170,8 @@
                 builder.append(box.raw_str())
                 state = iter.next(state)
         else:
-            builder.append_charpsize(self.implementation.get_storage(),
+            with self.implementation as storage:
+                builder.append_charpsize(storage,
                                      self.implementation.get_storage_size())
 
         state = space.newtuple([
@@ -1456,6 +1462,7 @@
     imag = GetSetProperty(W_NDimArray.descr_get_imag,
                           W_NDimArray.descr_set_imag),
     conj = interp2app(W_NDimArray.descr_conj),
+    conjugate = interp2app(W_NDimArray.descr_conj),
 
     argsort  = interp2app(W_NDimArray.descr_argsort),
     sort  = interp2app(W_NDimArray.descr_sort),
diff --git a/pypy/module/micronumpy/selection.py b/pypy/module/micronumpy/selection.py
--- a/pypy/module/micronumpy/selection.py
+++ b/pypy/module/micronumpy/selection.py
@@ -33,14 +33,14 @@
             self.values = values
             self.indexes = indexes
 
-        def getitem(self, item):
+        def getitem(self, idx):
             if count < 2:
-                v = raw_storage_getitem(TP, self.values, item * self.stride_size
+                v = raw_storage_getitem(TP, self.values, idx * self.stride_size
                                     + self.start)
             else:
                 v = []
                 for i in range(count):
-                    _v = raw_storage_getitem(TP, self.values, item * self.stride_size
+                    _v = raw_storage_getitem(TP, self.values, idx * self.stride_size
                                     + self.start + step * i)
                     v.append(_v)
             if comp_type == 'int':
@@ -52,7 +52,7 @@
             else:
                 raise NotImplementedError('cannot reach')
             return (v, raw_storage_getitem(lltype.Signed, self.indexes,
-                                           item * self.index_stride_size +
+                                           idx * self.index_stride_size +
                                            self.index_start))
 
         def setitem(self, idx, item):
@@ -134,37 +134,37 @@
         # create array of indexes
         dtype = descriptor.get_dtype_cache(space).w_longdtype
         index_arr = W_NDimArray.from_shape(space, arr.get_shape(), dtype)
-        storage = index_arr.implementation.get_storage()
-        if len(arr.get_shape()) == 1:
-            for i in range(arr.get_size()):
-                raw_storage_setitem(storage, i * INT_SIZE, i)
-            r = Repr(INT_SIZE, itemsize, arr.get_size(), arr.get_storage(),
-                     storage, 0, arr.start)
-            ArgSort(r).sort()

From noreply at buildbot.pypy.org  Wed Apr 22 12:20:57 2015
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Wed, 22 Apr 2015 12:20:57 +0200 (CEST)
Subject: [pypy-commit] pypy default: Converted README.md to reStructuredText
 inside rpython/doc/
Message-ID: <20150422102057.83A061C06CD@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: 
Changeset: r76878:9b7808006d44
Date: 2015-03-12 13:40 +0100
http://bitbucket.org/pypy/pypy/changeset/9b7808006d44/

Log:	Converted README.md to reStructuredText inside rpython/doc/ Added
	short section and reference inside rlib.rst.

diff --git a/rpython/doc/rlib.rst b/rpython/doc/rlib.rst
--- a/rpython/doc/rlib.rst
+++ b/rpython/doc/rlib.rst
@@ -128,6 +128,14 @@
 a hierarchy of Address classes, in a typical static-OO-programming style.
 
 
+rstrategies
+-----------
+
+The :source:`rpython/rlib/rstrategies` module contains a library to implement storage strategies in
+RPython VMs. The library is language-independent and extensible.
+More details and examples can be found in the :doc:`rstrategies documentation `.
+
+
 streamio
 --------
 
diff --git a/rpython/doc/rstrategies.rst b/rpython/doc/rstrategies.rst
new file mode 100644
--- /dev/null
+++ b/rpython/doc/rstrategies.rst
@@ -0,0 +1,148 @@
+rstrategies
+===========
+
+A library to implement storage strategies in VMs based on the RPython
+toolchain. rstrategies can be used in VMs for any language or language
+family.
+
+This library has been developed as part of a Masters Thesis by `Anton
+Gulenko `__.
+
+The original paper describing the optimization "Storage Strategies for
+collections in dynamically typed languages" by C.F. Bolz, L. Diekmann
+and L. Tratt can be found
+`here `__.
+
+So far, this library has been adpoted by 3 VMs:
+`RSqueak `__,
+`Topaz `__ (`Forked
+here `__) and
+`Pycket `__ (`Forked
+here `__).
+
+Concept
+^^^^^^^
+
+Collections are often used homogeneously, i.e. they contain only objects
+of the same type. Primitive numeric types like ints or floats are
+especially interesting for optimization. These cases can be optimized by
+storing the unboxed data of these objects in consecutive memory. This is
+done by letting a special "strategy" object handle the entire storage of
+a collection. The collection object holds two separate references: one
+to its strategy and one to its storage. Every operation on the
+collection is delegated to the strategy, which accesses the storage when
+needed. The strategy can be switched to a more suitable one, which might
+require converting the storage array.
+
+Usage
+-----
+
+The following are the steps needed to integrated rstrategies in an
+RPython VM. Because of the special nature of this library it is not
+enough to simply call some API methods; the library must be integrated
+within existing VM classes using a metaclass, mixins and other
+meta-programming techniques.
+
+The sequence of steps described here is something like a "setup
+walkthrough", and might be a bit abstract. To see a concrete example,
+look at
+`AbstractShadow `__,
+`StrategyFactory `__
+and
+`W\_PointersObject `__
+from the `RSqueak VM `__. The
+code is also well commented.
+
+Basics
+^^^^^^
+
+Currently the rstrategies library supports fixed sized and variable
+sized collections. This can be used to optimize a wide range of
+primitive data structures like arrays, lists or regular objects. Any of
+these are called 'collections' in this context. The VM should have a
+central class or class hierarchy for collections. In order to extend
+these classes and use strategies, the library needs accessor methods for
+two attributes of collection objects: strategy and storage. The easiest
+way is adding the following line to the body of the root collection
+class:
+
+::
+
+    rstrategies.make_accessors(strategy='strategy', storage='storage')
+
+This will generate the 4 accessor methods
+``_[get/set]_[storage/strategy]()`` for the respective attributes.
+Alternatively, implement these methods manually or overwrite the
+getters/setters in ``StrategyFactory``.
+
+Next, the strategy classes must be defined. This requires a small class
+hierarchy with a dedicated root class. In the definition of this root
+class, include the following lines:
+
+::
+
+        __metaclass__ = rstrategies.StrategyMetaclass
+        import_from_mixin(rstrategies.AbstractStrategy)
+        import_from_mixin(rstrategies.SafeIndexingMixin)
+
+``import_from_mixin`` can be found in ``rpython.rlib.objectmodel``. If
+index-checking is performed safely at other places in the VM, you can
+use ``rstrategies.UnsafeIndexingMixin`` instead. If you need your own
+metaclass, you can combine yours with the rstrategies one using multiple
+inheritance `like
+here `__.
+Also implement a ``storage_factory()`` method, which returns an instance
+of ``rstrategies.StorageFactory``, which is described below.
+
+Strategy classes
+^^^^^^^^^^^^^^^^
+
+Now you can create the actual strategy classes, subclassing them from
+the single root class. The following list summarizes the basic
+strategies available. \* ``EmptyStrategy`` A strategy for empty
+collections; very efficient, but limited. Does not allocate anything. \*
+``SingleValueStrategy`` A strategy for collections containing the same
+object ``n`` times. Only allocates memory to store the size of the
+collection. \* ``GenericStrategy`` A non-optimized strategy backed by a
+generic python list. This is the fallback strategy, since it can store
+everything, but is not optimized. \* ``WeakGenericStrategy`` Like
+``GenericStrategy``, but uses ``weakref`` to hold on weakly to its
+elements. \* ``SingleTypeStrategy`` Can store a single unboxed type like
+int or float. This is the main \* ``TaggingStrategy`` Extension of
+SingleTypeStrategy. Uses a specific value in the value range of the
+unboxed type to represent one additional, arbitrary object.
+
+There are also intermediate classes, which allow creating new, more
+customized strategies. For this, you should get familiar with the code.
+
+Include one of these mixin classes using ``import_from_mixin``. The
+mixin classes contain comments describing methods or fields which are
+also required in the strategy class in order to use them. Additionally,
+add the @rstrategies.strategy(generalize=alist) decorator to all
+strategy classes. The ``alist`` parameter must contain all strategies,
+which the decorated strategy can switch to, if it can not represent a
+new element anymore.
+`Example `__
+for an implemented strategy. See the other strategy classes behind this
+link for more examples.
+
+Strategy Factory
+^^^^^^^^^^^^^^^^
+
+The last part is subclassing ``rstrategies.StrategyFactory``,
+overwriting the method ``instantiate_strategy`` if necessary and passing
+the strategies root class to the constructor. The factory provides the
+methods ``switch_strategy``, ``set_initial_strategy``,
+``strategy_type_for`` which can be used by the VM code to use the
+mechanism behind strategies. See the comments in the source code.
+
+The strategy mixins offer the following methods to manipulate the
+contents of the collection: \* basic API \* ``size`` \* fixed size API
+\* ``store``, ``fetch``, ``slice``, ``store_all``, ``fetch_all`` \*
+variable size API \* ``insert``, ``delete``, ``append``, ``pop``
+
+If the collection has a fixed size, simply never use any of the variable
+size methods in the VM code. Since the strategies are singletons, these
+methods need the collection object as first parameter. For convenience,
+more fitting accessor methods should be implemented on the collection
+class itself.
diff --git a/rpython/rlib/rstrategies/README.md b/rpython/rlib/rstrategies/README.md
deleted file mode 100644
--- a/rpython/rlib/rstrategies/README.md
+++ /dev/null
@@ -1,101 +0,0 @@
-# rstrategies
-
-A library to implement storage strategies in VMs based on the RPython toolchain.
-rstrategies can be used in VMs for any language or language family.
-
-This library has been developed as part of a Masters Thesis by [Anton Gulenko](https://github.com/antongulenko).
-
-The original paper describing the optimization "Storage Strategies for collections in dynamically typed languages" by C.F. Bolz, L. Diekmann and L. Tratt can be found [here](http://stups.hhu.de/mediawiki/images/3/3b/Pub-BoDiTr13_246.pdf).
-
-So far, this library has been adpoted by 3 VMs: [RSqueak](https://github.com/HPI-SWA-Lab/RSqueak), [Topaz](https://github.com/topazproject/topaz) ([Forked here](https://github.com/antongulenko/topaz/tree/rstrategies)) and [Pycket](https://github.com/samth/pycket) ([Forked here](https://github.com/antongulenko/pycket/tree/rstrategies)).
-
-#### Concept
-
-Collections are often used homogeneously, i.e. they contain only objects of the same type.
-Primitive numeric types like ints or floats are especially interesting for optimization.
-These cases can be optimized by storing the unboxed data of these objects in consecutive memory.
-This is done by letting a special "strategy" object handle the entire storage of a collection.
-The collection object holds two separate references: one to its strategy and one to its storage.
-Every operation on the collection is delegated to the strategy, which accesses the storage when needed.
-The strategy can be switched to a more suitable one, which might require converting the storage array.
-
-## Usage
-
-The following are the steps needed to integrated rstrategies in an RPython VM.
-Because of the special nature of this library it is not enough to simply call some API methods; the library must be integrated within existing VM classes using a metaclass, mixins and other meta-programming techniques.
-
-The sequence of steps described here is something like a "setup walkthrough", and might be a bit abstract.
-To see a concrete example, look at [AbstractShadow](https://github.com/HPI-SWA-Lab/RSqueak/blob/d5ff2572106d23a5246884de6f8b86f46d85f4f7/spyvm/storage.py#L73), [StrategyFactory](https://github.com/HPI-SWA-Lab/RSqueak/blob/d5ff2572106d23a5246884de6f8b86f46d85f4f7/spyvm/storage.py#L126) and [W_PointersObject](https://github.com/HPI-SWA-Lab/RSqueak/blob/d5ff2572106d23a5246884de6f8b86f46d85f4f7/spyvm/model.py#L565) from the [RSqueak VM](https://github.com/HPI-SWA-Lab/RSqueak).
-The code is also well commented.
-
-#### Basics
-
-Currently the rstrategies library supports fixed sized and variable sized collections.
-This can be used to optimize a wide range of primitive data structures like arrays, lists or regular objects.
-Any of these are called 'collections' in this context.
-The VM should have a central class or class hierarchy for collections.
-In order to extend these classes and use strategies, the library needs accessor methods for two attributes of collection objects: strategy and storage.
-The easiest way is adding the following line to the body of the root collection class:
-```
-rstrategies.make_accessors(strategy='strategy', storage='storage')
-```
-This will generate the 4 accessor methods ```_[get/set]_[storage/strategy]()``` for the respective attributes.
-Alternatively, implement these methods manually or overwrite the getters/setters in ```StrategyFactory```.
-
-Next, the strategy classes must be defined. This requires a small class hierarchy with a dedicated root class.
-In the definition of this root class, include the following lines:
-```
-    __metaclass__ = rstrategies.StrategyMetaclass
-    import_from_mixin(rstrategies.AbstractStrategy)
-    import_from_mixin(rstrategies.SafeIndexingMixin)
-```
-
-```import_from_mixin``` can be found in ```rpython.rlib.objectmodel```.
-If index-checking is performed safely at other places in the VM, you can use ```rstrategies.UnsafeIndexingMixin``` instead.
-If you need your own metaclass, you can combine yours with the rstrategies one using multiple inheritance [like here](https://github.com/HPI-SWA-Lab/RSqueak/blob/d5ff2572106d23a5246884de6f8b86f46d85f4f7/spyvm/storage_contexts.py#L24).
-Also implement a ```storage_factory()``` method, which returns an instance of ```rstrategies.StorageFactory```, which is described below.
-
-#### Strategy classes
-
-Now you can create the actual strategy classes, subclassing them from the single root class.
-The following list summarizes the basic strategies available.
-* ```EmptyStrategy```
-    A strategy for empty collections; very efficient, but limited. Does not allocate anything.
-* ```SingleValueStrategy```
-    A strategy for collections containing the same object ```n``` times. Only allocates memory to store the size of the collection.
-* ```GenericStrategy```
-    A non-optimized strategy backed by a generic python list. This is the fallback strategy, since it can store everything, but is not optimized.
-* ```WeakGenericStrategy```
-    Like ```GenericStrategy```, but uses ```weakref``` to hold on weakly to its elements.
-* ```SingleTypeStrategy```
-    Can store a single unboxed type like int or float. This is the main 
-* ```TaggingStrategy```
-    Extension of SingleTypeStrategy. Uses a specific value in the value range of the unboxed type to represent
-    one additional, arbitrary object.
-
-There are also intermediate classes, which allow creating new, more customized strategies. For this, you should get familiar with the code.
-
-Include one of these mixin classes using ```import_from_mixin```.
-The mixin classes contain comments describing methods or fields which are also required in the strategy class in order to use them.
-Additionally, add the @rstrategies.strategy(generalize=alist) decorator to all strategy classes.
-The ```alist``` parameter must contain all strategies, which the decorated strategy can switch to, if it can not represent a new element anymore.
-[Example](https://github.com/HPI-SWA-Lab/RSqueak/blob/d5ff2572106d23a5246884de6f8b86f46d85f4f7/spyvm/storage.py#L87) for an implemented strategy.
-See the other strategy classes behind this link for more examples.
-
-#### Strategy Factory
-
-The last part is subclassing ```rstrategies.StrategyFactory```, overwriting the method ```instantiate_strategy``` if necessary and passing the strategies root class to the constructor.
-The factory provides the methods ```switch_strategy```, ```set_initial_strategy```, ```strategy_type_for``` which can be used by the VM code to use the mechanism behind strategies.
-See the comments in the source code.
-
-The strategy mixins offer the following methods to manipulate the contents of the collection:
-* basic API
-    * ```size```
-* fixed size API
-    * ```store```, ```fetch```, ```slice```, ```store_all```, ```fetch_all```
-* variable size API
-    * ```insert```, ```delete```, ```append```, ```pop```
-
-If the collection has a fixed size, simply never use any of the variable size methods in the VM code.
-Since the strategies are singletons, these methods need the collection object as first parameter.
-For convenience, more fitting accessor methods should be implemented on the collection class itself.

From noreply at buildbot.pypy.org  Wed Apr 22 12:20:58 2015
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Wed, 22 Apr 2015 12:20:58 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fixed formatting issues in doc file.
Message-ID: <20150422102058.96A5B1C06CD@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: 
Changeset: r76879:83b5d7b51f28
Date: 2015-03-12 14:52 +0100
http://bitbucket.org/pypy/pypy/changeset/83b5d7b51f28/

Log:	Fixed formatting issues in doc file. Added example code for the
	Usage-walkthrough.

diff --git a/rpython/doc/rstrategies.rst b/rpython/doc/rstrategies.rst
--- a/rpython/doc/rstrategies.rst
+++ b/rpython/doc/rstrategies.rst
@@ -21,7 +21,7 @@
 here `__).
 
 Concept
-^^^^^^^
+-------
 
 Collections are often used homogeneously, i.e. they contain only objects
 of the same type. Primitive numeric types like ints or floats are
@@ -35,7 +35,7 @@
 require converting the storage array.
 
 Usage
------
+~~~~~
 
 The following are the steps needed to integrated rstrategies in an
 RPython VM. Because of the special nature of this library it is not
@@ -46,15 +46,15 @@
 The sequence of steps described here is something like a "setup
 walkthrough", and might be a bit abstract. To see a concrete example,
 look at
-`AbstractShadow `__,
-`StrategyFactory `__
+`SingletonStorageStrategy `__,
+`StrategyFactory `__
 and
-`W\_PointersObject `__
+`W\_PointersObject `__
 from the `RSqueak VM `__. The
 code is also well commented.
 
 Basics
-^^^^^^
+-------
 
 Currently the rstrategies library supports fixed sized and variable
 sized collections. This can be used to optimize a wide range of
@@ -94,23 +94,37 @@
 Also implement a ``storage_factory()`` method, which returns an instance
 of ``rstrategies.StorageFactory``, which is described below.
 
+An example ``AbstractStrategy`` class, which also stores an additional ``space`` parameter could looks like this:
+
+::
+
+class AbstractStrategy(AbstractStrategy):
+    _attrs_ = ['space']
+    _immutable_fields_ = ['space']
+    __metaclass__ = rstrat.StrategyMetaclass
+    import_from_mixin(rstrat.AbstractStrategy)
+    import_from_mixin(rstrategies.SafeIndexingMixin)
+    
+    def __init__(self, space):
+        self.space = space
+    
+    def strategy_factory(self):
+        return self.space.strategy_factory
+
+
 Strategy classes
-^^^^^^^^^^^^^^^^
+----------------
 
 Now you can create the actual strategy classes, subclassing them from
 the single root class. The following list summarizes the basic
-strategies available. \* ``EmptyStrategy`` A strategy for empty
-collections; very efficient, but limited. Does not allocate anything. \*
-``SingleValueStrategy`` A strategy for collections containing the same
-object ``n`` times. Only allocates memory to store the size of the
-collection. \* ``GenericStrategy`` A non-optimized strategy backed by a
-generic python list. This is the fallback strategy, since it can store
-everything, but is not optimized. \* ``WeakGenericStrategy`` Like
-``GenericStrategy``, but uses ``weakref`` to hold on weakly to its
-elements. \* ``SingleTypeStrategy`` Can store a single unboxed type like
-int or float. This is the main \* ``TaggingStrategy`` Extension of
-SingleTypeStrategy. Uses a specific value in the value range of the
-unboxed type to represent one additional, arbitrary object.
+strategies available.
+
+- ``EmptyStrategy`` A strategy for empty collections; very efficient, but limited. Does not allocate anything.
+- ``SingleValueStrategy`` A strategy for collections containing the same object ``n`` times. Only allocates memory to store the size of the collection.
+- ``GenericStrategy`` A non-optimized strategy backed by a generic python list. This is the fallback strategy, since it can store everything, but is not optimized.
+- ``WeakGenericStrategy`` Like ``GenericStrategy``, but uses ``weakref`` to hold on weakly to its elements.
+- ``SingleTypeStrategy`` Can store a single unboxed type like int or float. This is the main optimizing strategy
+- ``TaggingStrategy`` Extension of SingleTypeStrategy. Uses a specific value in the value range of the unboxed type to represent one additional, arbitrary object. For example, one of ``float``'s ``NaN`` representations can be used to represent special value like ``nil``.
 
 There are also intermediate classes, which allow creating new, more
 customized strategies. For this, you should get familiar with the code.
@@ -118,7 +132,7 @@
 Include one of these mixin classes using ``import_from_mixin``. The
 mixin classes contain comments describing methods or fields which are
 also required in the strategy class in order to use them. Additionally,
-add the @rstrategies.strategy(generalize=alist) decorator to all
+add the ``@rstrategies.strategy(generalize=alist)`` decorator to all
 strategy classes. The ``alist`` parameter must contain all strategies,
 which the decorated strategy can switch to, if it can not represent a
 new element anymore.
@@ -126,8 +140,21 @@
 for an implemented strategy. See the other strategy classes behind this
 link for more examples.
 
+An example strategy class for optimized ``int`` storage could look like this:
+
+::
+
+ at rstrat.strategy(generalize=[GenericStrategy])
+class IntegerOrNilStrategy(AbstractStrategy):
+    import_from_mixin(rstrat.TaggingStrategy)
+    contained_type = model.W_Integer
+    def wrap(self, val): return self.space.wrap_int(val)
+    def unwrap(self, w_val): return self.space.unwrap_int(w_val)
+    def wrapped_tagged_value(self): return self.space.w_nil
+    def unwrapped_tagged_value(self): return constants.MAXINT
+
 Strategy Factory
-^^^^^^^^^^^^^^^^
+----------------
 
 The last part is subclassing ``rstrategies.StrategyFactory``,
 overwriting the method ``instantiate_strategy`` if necessary and passing
@@ -137,12 +164,46 @@
 mechanism behind strategies. See the comments in the source code.
 
 The strategy mixins offer the following methods to manipulate the
-contents of the collection: \* basic API \* ``size`` \* fixed size API
-\* ``store``, ``fetch``, ``slice``, ``store_all``, ``fetch_all`` \*
-variable size API \* ``insert``, ``delete``, ``append``, ``pop``
+contents of the collection:
+
+- basic API
+
+  - ``size``
+
+- fixed size API
+
+  - ``store``, ``fetch``, ``slice``, ``store_all``, ``fetch_all``
+
+- variable size API
+
+  - ``insert``, ``delete``, ``append``, ``pop``
 
 If the collection has a fixed size, simply never use any of the variable
 size methods in the VM code. Since the strategies are singletons, these
 methods need the collection object as first parameter. For convenience,
 more fitting accessor methods should be implemented on the collection
 class itself.
+
+An example strategy factory for the ``AbstractStrategy`` class above could look like this:
+
+::
+
+class StrategyFactory(rstrategies.StrategyFactory):
+    _attrs_ = ['space']
+    _immutable_fields_ = ['space']
+    
+    def __init__(self, space):
+        self.space = space
+        rstrat.StrategyFactory.__init__(self, AbstractStrategy)
+    
+    def instantiate_strategy(self, strategy_type):
+        return strategy_type(self.space)
+    
+    def strategy_type_for(self, list_w, weak=False):
+        """
+        Helper method for handling weak objects specially
+        """
+        if weak:
+            return WeakListStrategy
+        return rstrategies.StrategyFactory.strategy_type_for(self, list_w)
+    
\ No newline at end of file

From noreply at buildbot.pypy.org  Wed Apr 22 12:20:59 2015
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Wed, 22 Apr 2015 12:20:59 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fixed source-code formatting in doc
Message-ID: <20150422102059.AF3131C06CD@cobra.cs.uni-duesseldorf.de>

Author: anton_gulenko 
Branch: 
Changeset: r76880:7d7e19e7f237
Date: 2015-03-12 13:58 +0000
http://bitbucket.org/pypy/pypy/changeset/7d7e19e7f237/

Log:	Fixed source-code formatting in doc (rstrategies.rst edited online
	with Bitbucket)

diff --git a/rpython/doc/rstrategies.rst b/rpython/doc/rstrategies.rst
--- a/rpython/doc/rstrategies.rst
+++ b/rpython/doc/rstrategies.rst
@@ -98,18 +98,18 @@
 
 ::
 
-class AbstractStrategy(AbstractStrategy):
-    _attrs_ = ['space']
-    _immutable_fields_ = ['space']
-    __metaclass__ = rstrat.StrategyMetaclass
-    import_from_mixin(rstrat.AbstractStrategy)
-    import_from_mixin(rstrategies.SafeIndexingMixin)
-    
-    def __init__(self, space):
-        self.space = space
-    
-    def strategy_factory(self):
-        return self.space.strategy_factory
+    class AbstractStrategy(AbstractStrategy):
+        _attrs_ = ['space']
+        _immutable_fields_ = ['space']
+        __metaclass__ = rstrat.StrategyMetaclass
+        import_from_mixin(rstrat.AbstractStrategy)
+        import_from_mixin(rstrategies.SafeIndexingMixin)
+        
+        def __init__(self, space):
+            self.space = space
+        
+        def strategy_factory(self):
+            return self.space.strategy_factory
 
 
 Strategy classes
@@ -144,14 +144,14 @@
 
 ::
 
- at rstrat.strategy(generalize=[GenericStrategy])
-class IntegerOrNilStrategy(AbstractStrategy):
-    import_from_mixin(rstrat.TaggingStrategy)
-    contained_type = model.W_Integer
-    def wrap(self, val): return self.space.wrap_int(val)
-    def unwrap(self, w_val): return self.space.unwrap_int(w_val)
-    def wrapped_tagged_value(self): return self.space.w_nil
-    def unwrapped_tagged_value(self): return constants.MAXINT
+    @rstrat.strategy(generalize=[GenericStrategy])
+    class IntegerOrNilStrategy(AbstractStrategy):
+        import_from_mixin(rstrat.TaggingStrategy)
+        contained_type = model.W_Integer
+        def wrap(self, val): return self.space.wrap_int(val)
+        def unwrap(self, w_val): return self.space.unwrap_int(w_val)
+        def wrapped_tagged_value(self): return self.space.w_nil
+        def unwrapped_tagged_value(self): return constants.MAXINT
 
 Strategy Factory
 ----------------
@@ -188,22 +188,22 @@
 
 ::
 
-class StrategyFactory(rstrategies.StrategyFactory):
-    _attrs_ = ['space']
-    _immutable_fields_ = ['space']
-    
-    def __init__(self, space):
-        self.space = space
-        rstrat.StrategyFactory.__init__(self, AbstractStrategy)
-    
-    def instantiate_strategy(self, strategy_type):
-        return strategy_type(self.space)
-    
-    def strategy_type_for(self, list_w, weak=False):
-        """
-        Helper method for handling weak objects specially
-        """
-        if weak:
-            return WeakListStrategy
+    class StrategyFactory(rstrategies.StrategyFactory):
+        _attrs_ = ['space']
+        _immutable_fields_ = ['space']
+        
+        def __init__(self, space):
+            self.space = space
+            rstrat.StrategyFactory.__init__(self, AbstractStrategy)
+        
+        def instantiate_strategy(self, strategy_type):
+            return strategy_type(self.space)
+        
+        def strategy_type_for(self, list_w, weak=False):
+            """
+            Helper method for handling weak objects specially
+            """
+            if weak:
+                return WeakListStrategy
         return rstrategies.StrategyFactory.strategy_type_for(self, list_w)
     
\ No newline at end of file

From noreply at buildbot.pypy.org  Wed Apr 22 12:21:00 2015
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Wed, 22 Apr 2015 12:21:00 +0200 (CEST)
Subject: [pypy-commit] pypy default: Removed accidentally committed coverage
	report files.
Message-ID: <20150422102100.E5B8F1C06CD@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: 
Changeset: r76881:063c4e7456b4
Date: 2015-03-12 15:02 +0100
http://bitbucket.org/pypy/pypy/changeset/063c4e7456b4/

Log:	Removed accidentally committed coverage report files.

diff too long, truncating to 2000 out of 5357 lines

diff --git a/rpython/rlib/rstrategies/.coverage b/rpython/rlib/rstrategies/.coverage
deleted file mode 100644
Binary file rpython/rlib/rstrategies/.coverage has changed
diff --git a/rpython/rlib/rstrategies/htmlcov1/coverage_html.js b/rpython/rlib/rstrategies/htmlcov1/coverage_html.js
deleted file mode 100644
--- a/rpython/rlib/rstrategies/htmlcov1/coverage_html.js
+++ /dev/null
@@ -1,376 +0,0 @@
-// Coverage.py HTML report browser code.
-/*jslint browser: true, sloppy: true, vars: true, plusplus: true, maxerr: 50, indent: 4 */
-/*global coverage: true, document, window, $ */
-
-coverage = {};
-
-// Find all the elements with shortkey_* class, and use them to assign a shotrtcut key.
-coverage.assign_shortkeys = function () {
-    $("*[class*='shortkey_']").each(function (i, e) {
-        $.each($(e).attr("class").split(" "), function (i, c) {
-            if (/^shortkey_/.test(c)) {
-                $(document).bind('keydown', c.substr(9), function () {
-                    $(e).click();
-                });
-            }
-        });
-    });
-};
-
-// Create the events for the help panel.
-coverage.wire_up_help_panel = function () {
-    $("#keyboard_icon").click(function () {
-        // Show the help panel, and position it so the keyboard icon in the
-        // panel is in the same place as the keyboard icon in the header.
-        $(".help_panel").show();
-        var koff = $("#keyboard_icon").offset();
-        var poff = $("#panel_icon").position();
-        $(".help_panel").offset({
-            top: koff.top-poff.top,
-            left: koff.left-poff.left
-        });
-    });
-    $("#panel_icon").click(function () {
-        $(".help_panel").hide();
-    });
-};
-
-// Loaded on index.html
-coverage.index_ready = function ($) {
-    // Look for a cookie containing previous sort settings:
-    var sort_list = [];
-    var cookie_name = "COVERAGE_INDEX_SORT";
-    var i;
-
-    // This almost makes it worth installing the jQuery cookie plugin:
-    if (document.cookie.indexOf(cookie_name) > -1) {
-        var cookies = document.cookie.split(";");
-        for (i = 0; i < cookies.length; i++) {
-            var parts = cookies[i].split("=");
-
-            if ($.trim(parts[0]) === cookie_name && parts[1]) {
-                sort_list = eval("[[" + parts[1] + "]]");
-                break;
-            }
-        }
-    }
-
-    // Create a new widget which exists only to save and restore
-    // the sort order:
-    $.tablesorter.addWidget({
-        id: "persistentSort",
-
-        // Format is called by the widget before displaying:
-        format: function (table) {
-            if (table.config.sortList.length === 0 && sort_list.length > 0) {
-                // This table hasn't been sorted before - we'll use
-                // our stored settings:
-                $(table).trigger('sorton', [sort_list]);
-            }
-            else {
-                // This is not the first load - something has
-                // already defined sorting so we'll just update
-                // our stored value to match:
-                sort_list = table.config.sortList;
-            }
-        }
-    });
-
-    // Configure our tablesorter to handle the variable number of
-    // columns produced depending on report options:
-    var headers = [];
-    var col_count = $("table.index > thead > tr > th").length;
-
-    headers[0] = { sorter: 'text' };
-    for (i = 1; i < col_count-1; i++) {
-        headers[i] = { sorter: 'digit' };
-    }
-    headers[col_count-1] = { sorter: 'percent' };
-
-    // Enable the table sorter:
-    $("table.index").tablesorter({
-        widgets: ['persistentSort'],
-        headers: headers
-    });
-
-    coverage.assign_shortkeys();
-    coverage.wire_up_help_panel();
-
-    // Watch for page unload events so we can save the final sort settings:
-    $(window).unload(function () {
-        document.cookie = cookie_name + "=" + sort_list.toString() + "; path=/";
-    });
-};
-
-// -- pyfile stuff --
-
-coverage.pyfile_ready = function ($) {
-    // If we're directed to a particular line number, highlight the line.
-    var frag = location.hash;
-    if (frag.length > 2 && frag[1] === 'n') {
-        $(frag).addClass('highlight');
-        coverage.set_sel(parseInt(frag.substr(2), 10));
-    }
-    else {
-        coverage.set_sel(0);
-    }
-
-    $(document)
-        .bind('keydown', 'j', coverage.to_next_chunk_nicely)
-        .bind('keydown', 'k', coverage.to_prev_chunk_nicely)
-        .bind('keydown', '0', coverage.to_top)
-        .bind('keydown', '1', coverage.to_first_chunk)
-        ;
-
-    $(".button_toggle_run").click(function (evt) {coverage.toggle_lines(evt.target, "run");});
-    $(".button_toggle_exc").click(function (evt) {coverage.toggle_lines(evt.target, "exc");});
-    $(".button_toggle_mis").click(function (evt) {coverage.toggle_lines(evt.target, "mis");});
-    $(".button_toggle_par").click(function (evt) {coverage.toggle_lines(evt.target, "par");});
-
-    coverage.assign_shortkeys();
-    coverage.wire_up_help_panel();
-};
-
-coverage.toggle_lines = function (btn, cls) {
-    btn = $(btn);
-    var hide = "hide_"+cls;
-    if (btn.hasClass(hide)) {
-        $("#source ."+cls).removeClass(hide);
-        btn.removeClass(hide);
-    }
-    else {
-        $("#source ."+cls).addClass(hide);
-        btn.addClass(hide);
-    }
-};
-
-// Return the nth line div.
-coverage.line_elt = function (n) {
-    return $("#t" + n);
-};
-
-// Return the nth line number div.
-coverage.num_elt = function (n) {
-    return $("#n" + n);
-};
-
-// Return the container of all the code.
-coverage.code_container = function () {
-    return $(".linenos");
-};
-
-// Set the selection.  b and e are line numbers.
-coverage.set_sel = function (b, e) {
-    // The first line selected.
-    coverage.sel_begin = b;
-    // The next line not selected.
-    coverage.sel_end = (e === undefined) ? b+1 : e;
-};
-
-coverage.to_top = function () {
-    coverage.set_sel(0, 1);
-    coverage.scroll_window(0);
-};
-
-coverage.to_first_chunk = function () {
-    coverage.set_sel(0, 1);
-    coverage.to_next_chunk();
-};
-
-coverage.is_transparent = function (color) {
-    // Different browsers return different colors for "none".
-    return color === "transparent" || color === "rgba(0, 0, 0, 0)";
-};
-
-coverage.to_next_chunk = function () {
-    var c = coverage;
-
-    // Find the start of the next colored chunk.
-    var probe = c.sel_end;
-    while (true) {
-        var probe_line = c.line_elt(probe);
-        if (probe_line.length === 0) {
-            return;
-        }
-        var color = probe_line.css("background-color");
-        if (!c.is_transparent(color)) {
-            break;
-        }
-        probe++;
-    }
-
-    // There's a next chunk, `probe` points to it.
-    var begin = probe;
-
-    // Find the end of this chunk.
-    var next_color = color;
-    while (next_color === color) {
-        probe++;
-        probe_line = c.line_elt(probe);
-        next_color = probe_line.css("background-color");
-    }
-    c.set_sel(begin, probe);
-    c.show_selection();
-};
-
-coverage.to_prev_chunk = function () {
-    var c = coverage;
-
-    // Find the end of the prev colored chunk.
-    var probe = c.sel_begin-1;
-    var probe_line = c.line_elt(probe);
-    if (probe_line.length === 0) {
-        return;
-    }
-    var color = probe_line.css("background-color");
-    while (probe > 0 && c.is_transparent(color)) {
-        probe--;
-        probe_line = c.line_elt(probe);
-        if (probe_line.length === 0) {
-            return;
-        }
-        color = probe_line.css("background-color");
-    }
-
-    // There's a prev chunk, `probe` points to its last line.
-    var end = probe+1;
-
-    // Find the beginning of this chunk.
-    var prev_color = color;
-    while (prev_color === color) {
-        probe--;
-        probe_line = c.line_elt(probe);
-        prev_color = probe_line.css("background-color");
-    }
-    c.set_sel(probe+1, end);
-    c.show_selection();
-};
-
-// Return the line number of the line nearest pixel position pos
-coverage.line_at_pos = function (pos) {
-    var l1 = coverage.line_elt(1),
-        l2 = coverage.line_elt(2),
-        result;
-    if (l1.length && l2.length) {
-        var l1_top = l1.offset().top,
-            line_height = l2.offset().top - l1_top,
-            nlines = (pos - l1_top) / line_height;
-        if (nlines < 1) {
-            result = 1;
-        }
-        else {
-            result = Math.ceil(nlines);
-        }
-    }
-    else {
-        result = 1;
-    }
-    return result;
-};
-
-// Returns 0, 1, or 2: how many of the two ends of the selection are on
-// the screen right now?
-coverage.selection_ends_on_screen = function () {
-    if (coverage.sel_begin === 0) {
-        return 0;
-    }
-
-    var top = coverage.line_elt(coverage.sel_begin);
-    var next = coverage.line_elt(coverage.sel_end-1);
-
-    return (
-        (top.isOnScreen() ? 1 : 0) +
-        (next.isOnScreen() ? 1 : 0)
-    );
-};
-
-coverage.to_next_chunk_nicely = function () {
-    coverage.finish_scrolling();
-    if (coverage.selection_ends_on_screen() === 0) {
-        // The selection is entirely off the screen: select the top line on
-        // the screen.
-        var win = $(window);
-        coverage.select_line_or_chunk(coverage.line_at_pos(win.scrollTop()));
-    }
-    coverage.to_next_chunk();
-};
-
-coverage.to_prev_chunk_nicely = function () {
-    coverage.finish_scrolling();
-    if (coverage.selection_ends_on_screen() === 0) {
-        var win = $(window);
-        coverage.select_line_or_chunk(coverage.line_at_pos(win.scrollTop() + win.height()));
-    }
-    coverage.to_prev_chunk();
-};
-
-// Select line number lineno, or if it is in a colored chunk, select the
-// entire chunk
-coverage.select_line_or_chunk = function (lineno) {
-    var c = coverage;
-    var probe_line = c.line_elt(lineno);
-    if (probe_line.length === 0) {
-        return;
-    }
-    var the_color = probe_line.css("background-color");
-    if (!c.is_transparent(the_color)) {
-        // The line is in a highlighted chunk.
-        // Search backward for the first line.
-        var probe = lineno;
-        var color = the_color;
-        while (probe > 0 && color === the_color) {
-            probe--;
-            probe_line = c.line_elt(probe);
-            if (probe_line.length === 0) {
-                break;
-            }
-            color = probe_line.css("background-color");
-        }
-        var begin = probe + 1;
-
-        // Search forward for the last line.
-        probe = lineno;
-        color = the_color;
-        while (color === the_color) {
-            probe++;
-            probe_line = c.line_elt(probe);
-            color = probe_line.css("background-color");
-        }
-
-        coverage.set_sel(begin, probe);
-    }
-    else {
-        coverage.set_sel(lineno);
-    }
-};
-
-coverage.show_selection = function () {
-    var c = coverage;
-
-    // Highlight the lines in the chunk
-    c.code_container().find(".highlight").removeClass("highlight");
-    for (var probe = c.sel_begin; probe > 0 && probe < c.sel_end; probe++) {
-        c.num_elt(probe).addClass("highlight");
-    }
-
-    c.scroll_to_selection();
-};
-
-coverage.scroll_to_selection = function () {
-    // Scroll the page if the chunk isn't fully visible.
-    if (coverage.selection_ends_on_screen() < 2) {
-        // Need to move the page. The html,body trick makes it scroll in all
-        // browsers, got it from http://stackoverflow.com/questions/3042651
-        var top = coverage.line_elt(coverage.sel_begin);
-        var top_pos = parseInt(top.offset().top, 10);
-        coverage.scroll_window(top_pos - 30);
-    }
-};
-
-coverage.scroll_window = function (to_pos) {
-    $("html,body").animate({scrollTop: to_pos}, 200);
-};
-
-coverage.finish_scrolling = function () {
-    $("html,body").stop(true, true);
-};
diff --git a/rpython/rlib/rstrategies/htmlcov1/index.html b/rpython/rlib/rstrategies/htmlcov1/index.html
deleted file mode 100644
--- a/rpython/rlib/rstrategies/htmlcov1/index.html
+++ /dev/null
@@ -1,99 +0,0 @@
-
-
-
-    
-    Coverage report
-    
-    
-    
-    
-    
-    
-    
-
-
-
-
-    
-        Coverage report:
-            95%
-        
-        
-    
-
-
-
-    
-    Hot-keys on this page
-    
-    
-        n
-        s
-        m
-        x
-        
-        c   change column sorting
-    
-    
-
-
-
-    
-        
-            
-            
-                
-                
-                
-                
-                
-                
-            
-        
-        
-        
-            
-                
-                
-                
-                
-                
-                
-            
-        
-        
-            
-            
-                
-                
-                
-                
-                
-                
-            
-            
-            
-                
-                
-                
-                
-                
-                
-            
-            
-        
-    Module statements missing excluded coverage
Total 395 18 0 95%logger 46 2 0 96%
rstrategies 349 16 0 95%
-
-
-
-    
-        
-            coverage.py v3.7.1
-        
-    
-
-
-
-
diff --git a/rpython/rlib/rstrategies/htmlcov1/jquery.hotkeys.js b/rpython/rlib/rstrategies/htmlcov1/jquery.hotkeys.js
deleted file mode 100644
--- a/rpython/rlib/rstrategies/htmlcov1/jquery.hotkeys.js
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * jQuery Hotkeys Plugin
- * Copyright 2010, John Resig
- * Dual licensed under the MIT or GPL Version 2 licenses.
- *
- * Based upon the plugin by Tzury Bar Yochay:
- * http://github.com/tzuryby/hotkeys
- *
- * Original idea by:
- * Binny V A, http://www.openjs.com/scripts/events/keyboard_shortcuts/
-*/
-
-(function(jQuery){
-
-	jQuery.hotkeys = {
-		version: "0.8",
-
-		specialKeys: {
-			8: "backspace", 9: "tab", 13: "return", 16: "shift", 17: "ctrl", 18: "alt", 19: "pause",
-			20: "capslock", 27: "esc", 32: "space", 33: "pageup", 34: "pagedown", 35: "end", 36: "home",
-			37: "left", 38: "up", 39: "right", 40: "down", 45: "insert", 46: "del",
-			96: "0", 97: "1", 98: "2", 99: "3", 100: "4", 101: "5", 102: "6", 103: "7",
-			104: "8", 105: "9", 106: "*", 107: "+", 109: "-", 110: ".", 111 : "/",
-			112: "f1", 113: "f2", 114: "f3", 115: "f4", 116: "f5", 117: "f6", 118: "f7", 119: "f8",
-			120: "f9", 121: "f10", 122: "f11", 123: "f12", 144: "numlock", 145: "scroll", 191: "/", 224: "meta"
-		},
-
-		shiftNums: {
-			"`": "~", "1": "!", "2": "@", "3": "#", "4": "$", "5": "%", "6": "^", "7": "&",
-			"8": "*", "9": "(", "0": ")", "-": "_", "=": "+", ";": ": ", "'": "\"", ",": "<",
-			".": ">",  "/": "?",  "\\": "|"
-		}
-	};
-
-	function keyHandler( handleObj ) {
-		// Only care when a possible input has been specified
-		if ( typeof handleObj.data !== "string" ) {
-			return;
-		}
-
-		var origHandler = handleObj.handler,
-			keys = handleObj.data.toLowerCase().split(" ");
-
-		handleObj.handler = function( event ) {
-			// Don't fire in text-accepting inputs that we didn't directly bind to
-			if ( this !== event.target && (/textarea|select/i.test( event.target.nodeName ) ||
-				 event.target.type === "text") ) {
-				return;
-			}
-
-			// Keypress represents characters, not special keys
-			var special = event.type !== "keypress" && jQuery.hotkeys.specialKeys[ event.which ],
-				character = String.fromCharCode( event.which ).toLowerCase(),
-				key, modif = "", possible = {};
-
-			// check combinations (alt|ctrl|shift+anything)
-			if ( event.altKey && special !== "alt" ) {
-				modif += "alt+";
-			}
-
-			if ( event.ctrlKey && special !== "ctrl" ) {
-				modif += "ctrl+";
-			}
-
-			// TODO: Need to make sure this works consistently across platforms
-			if ( event.metaKey && !event.ctrlKey && special !== "meta" ) {
-				modif += "meta+";
-			}
-
-			if ( event.shiftKey && special !== "shift" ) {
-				modif += "shift+";
-			}
-
-			if ( special ) {
-				possible[ modif + special ] = true;
-
-			} else {
-				possible[ modif + character ] = true;
-				possible[ modif + jQuery.hotkeys.shiftNums[ character ] ] = true;
-
-				// "$" can be triggered as "Shift+4" or "Shift+$" or just "$"
-				if ( modif === "shift+" ) {
-					possible[ jQuery.hotkeys.shiftNums[ character ] ] = true;
-				}
-			}
-
-			for ( var i = 0, l = keys.length; i < l; i++ ) {
-				if ( possible[ keys[i] ] ) {
-					return origHandler.apply( this, arguments );
-				}
-			}
-		};
-	}
-
-	jQuery.each([ "keydown", "keyup", "keypress" ], function() {
-		jQuery.event.special[ this ] = { add: keyHandler };
-	});
-
-})( jQuery );
diff --git a/rpython/rlib/rstrategies/htmlcov1/jquery.isonscreen.js b/rpython/rlib/rstrategies/htmlcov1/jquery.isonscreen.js
deleted file mode 100644
--- a/rpython/rlib/rstrategies/htmlcov1/jquery.isonscreen.js
+++ /dev/null
@@ -1,53 +0,0 @@
-/* Copyright (c) 2010
- * @author Laurence Wheway
- * Dual licensed under the MIT (http://www.opensource.org/licenses/mit-license.php)
- * and GPL (http://www.opensource.org/licenses/gpl-license.php) licenses.
- *
- * @version 1.2.0
- */
-(function($) {
-	jQuery.extend({
-		isOnScreen: function(box, container) {
-			//ensure numbers come in as intgers (not strings) and remove 'px' is it's there
-			for(var i in box){box[i] = parseFloat(box[i])};
-			for(var i in container){container[i] = parseFloat(container[i])};
-
-			if(!container){
-				container = {
-					left: $(window).scrollLeft(),
-					top: $(window).scrollTop(),
-					width: $(window).width(),
-					height: $(window).height()
-				}
-			}
-
-			if(	box.left+box.width-container.left > 0 &&
-				box.left < container.width+container.left &&
-				box.top+box.height-container.top > 0 &&
-				box.top < container.height+container.top
-			) return true;
-			return false;
-		}
-	})
-
-
-	jQuery.fn.isOnScreen = function (container) {
-		for(var i in container){container[i] = parseFloat(container[i])};
-
-		if(!container){
-			container = {
-				left: $(window).scrollLeft(),
-				top: $(window).scrollTop(),
-				width: $(window).width(),
-				height: $(window).height()
-			}
-		}
-
-		if(	$(this).offset().left+$(this).width()-container.left > 0 &&
-			$(this).offset().left < container.width+container.left &&
-			$(this).offset().top+$(this).height()-container.top > 0 &&
-			$(this).offset().top < container.height+container.top
-		) return true;
-		return false;
-	}
-})(jQuery);
diff --git a/rpython/rlib/rstrategies/htmlcov1/jquery.min.js b/rpython/rlib/rstrategies/htmlcov1/jquery.min.js
deleted file mode 100644
--- a/rpython/rlib/rstrategies/htmlcov1/jquery.min.js
+++ /dev/null
@@ -1,166 +0,0 @@
-/*!
- * jQuery JavaScript Library v1.4.3
- * http://jquery.com/
- *
- * Copyright 2010, John Resig
- * Dual licensed under the MIT or GPL Version 2 licenses.
- * http://jquery.org/license
- *
- * Includes Sizzle.js
- * http://sizzlejs.com/
- * Copyright 2010, The Dojo Foundation
- * Released under the MIT, BSD, and GPL Licenses.
- *
- * Date: Thu Oct 14 23:10:06 2010 -0400
- */
-(function(E,A){function U(){return false}function ba(){return true}function ja(a,b,d){d[0].type=a;return c.event.handle.apply(b,d)}function Ga(a){var b,d,e=[],f=[],h,k,l,n,s,v,B,D;k=c.data(this,this.nodeType?"events":"__events__");if(typeof k==="function")k=k.events;if(!(a.liveFired===this||!k||!k.live||a.button&&a.type==="click")){if(a.namespace)D=RegExp("(^|\\.)"+a.namespace.split(".").join("\\.(?:.*\\.)?")+"(\\.|$)");a.liveFired=this;var H=k.live.slice(0);for(n=0;nd)break;a.currentTarget=f.elem;a.data=f.handleObj.data;
-a.handleObj=f.handleObj;D=f.handleObj.origHandler.apply(f.elem,arguments);if(D===false||a.isPropagationStopped()){d=f.level;if(D===false)b=false}}return b}}function Y(a,b){return(a&&a!=="*"?a+".":"")+b.replace(Ha,"`").replace(Ia,"&")}function ka(a,b,d){if(c.isFunction(b))return c.grep(a,function(f,h){return!!b.call(f,h,f)===d});else if(b.nodeType)return c.grep(a,function(f){return f===b===d});else if(typeof b==="string"){var e=c.grep(a,function(f){return f.nodeType===1});if(Ja.test(b))return c.filter(b,
-e,!d);else b=c.filter(b,e)}return c.grep(a,function(f){return c.inArray(f,b)>=0===d})}function la(a,b){var d=0;b.each(function(){if(this.nodeName===(a[d]&&a[d].nodeName)){var e=c.data(a[d++]),f=c.data(this,e);if(e=e&&e.events){delete f.handle;f.events={};for(var h in e)for(var k in e[h])c.event.add(this,h,e[h][k],e[h][k].data)}}})}function Ka(a,b){b.src?c.ajax({url:b.src,async:false,dataType:"script"}):c.globalEval(b.text||b.textContent||b.innerHTML||"");b.parentNode&&b.parentNode.removeChild(b)}
-function ma(a,b,d){var e=b==="width"?a.offsetWidth:a.offsetHeight;if(d==="border")return e;c.each(b==="width"?La:Ma,function(){d||(e-=parseFloat(c.css(a,"padding"+this))||0);if(d==="margin")e+=parseFloat(c.css(a,"margin"+this))||0;else e-=parseFloat(c.css(a,"border"+this+"Width"))||0});return e}function ca(a,b,d,e){if(c.isArray(b)&&b.length)c.each(b,function(f,h){d||Na.test(a)?e(a,h):ca(a+"["+(typeof h==="object"||c.isArray(h)?f:"")+"]",h,d,e)});else if(!d&&b!=null&&typeof b==="object")c.isEmptyObject(b)?
-e(a,""):c.each(b,function(f,h){ca(a+"["+f+"]",h,d,e)});else e(a,b)}function S(a,b){var d={};c.each(na.concat.apply([],na.slice(0,b)),function(){d[this]=a});return d}function oa(a){if(!da[a]){var b=c("<"+a+">").appendTo("body"),d=b.css("display");b.remove();if(d==="none"||d==="")d="block";da[a]=d}return da[a]}function ea(a){return c.isWindow(a)?a:a.nodeType===9?a.defaultView||a.parentWindow:false}var u=E.document,c=function(){function a(){if(!b.isReady){try{u.documentElement.doScroll("left")}catch(i){setTimeout(a,
-1);return}b.ready()}}var b=function(i,r){return new b.fn.init(i,r)},d=E.jQuery,e=E.$,f,h=/^(?:[^<]*(<[\w\W]+>)[^>]*$|#([\w\-]+)$)/,k=/\S/,l=/^\s+/,n=/\s+$/,s=/\W/,v=/\d/,B=/^<(\w+)\s*\/?>(?:<\/\1>)?$/,D=/^[\],:{}\s]*$/,H=/\\(?:["\\\/bfnrt]|u[0-9a-fA-F]{4})/g,w=/"[^"\\\n\r]*"|true|false|null|-?\d+(?:\.\d*)?(?:[eE][+\-]?\d+)?/g,G=/(?:^|:|,)(?:\s*\[)+/g,M=/(webkit)[ \/]([\w.]+)/,g=/(opera)(?:.*version)?[ \/]([\w.]+)/,j=/(msie) ([\w.]+)/,o=/(mozilla)(?:.*? rv:([\w.]+))?/,m=navigator.userAgent,p=false,
-q=[],t,x=Object.prototype.toString,C=Object.prototype.hasOwnProperty,P=Array.prototype.push,N=Array.prototype.slice,R=String.prototype.trim,Q=Array.prototype.indexOf,L={};b.fn=b.prototype={init:function(i,r){var y,z,F;if(!i)return this;if(i.nodeType){this.context=this[0]=i;this.length=1;return this}if(i==="body"&&!r&&u.body){this.context=u;this[0]=u.body;this.selector="body";this.length=1;return this}if(typeof i==="string")if((y=h.exec(i))&&(y[1]||!r))if(y[1]){F=r?r.ownerDocument||r:u;if(z=B.exec(i))if(b.isPlainObject(r)){i=
-[u.createElement(z[1])];b.fn.attr.call(i,r,true)}else i=[F.createElement(z[1])];else{z=b.buildFragment([y[1]],[F]);i=(z.cacheable?z.fragment.cloneNode(true):z.fragment).childNodes}return b.merge(this,i)}else{if((z=u.getElementById(y[2]))&&z.parentNode){if(z.id!==y[2])return f.find(i);this.length=1;this[0]=z}this.context=u;this.selector=i;return this}else if(!r&&!s.test(i)){this.selector=i;this.context=u;i=u.getElementsByTagName(i);return b.merge(this,i)}else return!r||r.jquery?(r||f).find(i):b(r).find(i);
-else if(b.isFunction(i))return f.ready(i);if(i.selector!==A){this.selector=i.selector;this.context=i.context}return b.makeArray(i,this)},selector:"",jquery:"1.4.3",length:0,size:function(){return this.length},toArray:function(){return N.call(this,0)},get:function(i){return i==null?this.toArray():i<0?this.slice(i)[0]:this[i]},pushStack:function(i,r,y){var z=b();b.isArray(i)?P.apply(z,i):b.merge(z,i);z.prevObject=this;z.context=this.context;if(r==="find")z.selector=this.selector+(this.selector?" ":
-"")+y;else if(r)z.selector=this.selector+"."+r+"("+y+")";return z},each:function(i,r){return b.each(this,i,r)},ready:function(i){b.bindReady();if(b.isReady)i.call(u,b);else q&&q.push(i);return this},eq:function(i){return i===-1?this.slice(i):this.slice(i,+i+1)},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},slice:function(){return this.pushStack(N.apply(this,arguments),"slice",N.call(arguments).join(","))},map:function(i){return this.pushStack(b.map(this,function(r,y){return i.call(r,
-y,r)}))},end:function(){return this.prevObject||b(null)},push:P,sort:[].sort,splice:[].splice};b.fn.init.prototype=b.fn;b.extend=b.fn.extend=function(){var i=arguments[0]||{},r=1,y=arguments.length,z=false,F,I,K,J,fa;if(typeof i==="boolean"){z=i;i=arguments[1]||{};r=2}if(typeof i!=="object"&&!b.isFunction(i))i={};if(y===r){i=this;--r}for(;r0)){if(q){for(var r=0;i=q[r++];)i.call(u,b);q=null}b.fn.triggerHandler&&b(u).triggerHandler("ready")}}},bindReady:function(){if(!p){p=true;if(u.readyState==="complete")return setTimeout(b.ready,
-1);if(u.addEventListener){u.addEventListener("DOMContentLoaded",t,false);E.addEventListener("load",b.ready,false)}else if(u.attachEvent){u.attachEvent("onreadystatechange",t);E.attachEvent("onload",b.ready);var i=false;try{i=E.frameElement==null}catch(r){}u.documentElement.doScroll&&i&&a()}}},isFunction:function(i){return b.type(i)==="function"},isArray:Array.isArray||function(i){return b.type(i)==="array"},isWindow:function(i){return i&&typeof i==="object"&&"setInterval"in i},isNaN:function(i){return i==
-null||!v.test(i)||isNaN(i)},type:function(i){return i==null?String(i):L[x.call(i)]||"object"},isPlainObject:function(i){if(!i||b.type(i)!=="object"||i.nodeType||b.isWindow(i))return false;if(i.constructor&&!C.call(i,"constructor")&&!C.call(i.constructor.prototype,"isPrototypeOf"))return false;for(var r in i);return r===A||C.call(i,r)},isEmptyObject:function(i){for(var r in i)return false;return true},error:function(i){throw i;},parseJSON:function(i){if(typeof i!=="string"||!i)return null;i=b.trim(i);
-if(D.test(i.replace(H,"@").replace(w,"]").replace(G,"")))return E.JSON&&E.JSON.parse?E.JSON.parse(i):(new Function("return "+i))();else b.error("Invalid JSON: "+i)},noop:function(){},globalEval:function(i){if(i&&k.test(i)){var r=u.getElementsByTagName("head")[0]||u.documentElement,y=u.createElement("script");y.type="text/javascript";if(b.support.scriptEval)y.appendChild(u.createTextNode(i));else y.text=i;r.insertBefore(y,r.firstChild);r.removeChild(y)}},nodeName:function(i,r){return i.nodeName&&i.nodeName.toUpperCase()===
-r.toUpperCase()},each:function(i,r,y){var z,F=0,I=i.length,K=I===A||b.isFunction(i);if(y)if(K)for(z in i){if(r.apply(i[z],y)===false)break}else for(;F";a=u.createDocumentFragment();a.appendChild(d.firstChild);c.support.checkClone=a.cloneNode(true).cloneNode(true).lastChild.checked;c(function(){var s=u.createElement("div");
-s.style.width=s.style.paddingLeft="1px";u.body.appendChild(s);c.boxModel=c.support.boxModel=s.offsetWidth===2;if("zoom"in s.style){s.style.display="inline";s.style.zoom=1;c.support.inlineBlockNeedsLayout=s.offsetWidth===2;s.style.display="";s.innerHTML="
";c.support.shrinkWrapBlocks=s.offsetWidth!==2}s.innerHTML=" t";var v=s.getElementsByTagName("td");c.support.reliableHiddenOffsets=v[0].offsetHeight===
-0;v[0].style.display="";v[1].style.display="none";c.support.reliableHiddenOffsets=c.support.reliableHiddenOffsets&&v[0].offsetHeight===0;s.innerHTML="";u.body.removeChild(s).style.display="none"});a=function(s){var v=u.createElement("div");s="on"+s;var B=s in v;if(!B){v.setAttribute(s,"return;");B=typeof v[s]==="function"}return B};c.support.submitBubbles=a("submit");c.support.changeBubbles=a("change");a=b=d=f=h=null}})();c.props={"for":"htmlFor","class":"className",readonly:"readOnly",maxlength:"maxLength",
-cellspacing:"cellSpacing",rowspan:"rowSpan",colspan:"colSpan",tabindex:"tabIndex",usemap:"useMap",frameborder:"frameBorder"};var pa={},Oa=/^(?:\{.*\}|\[.*\])$/;c.extend({cache:{},uuid:0,expando:"jQuery"+c.now(),noData:{embed:true,object:"clsid:D27CDB6E-AE6D-11cf-96B8-444553540000",applet:true},data:function(a,b,d){if(c.acceptData(a)){a=a==E?pa:a;var e=a.nodeType,f=e?a[c.expando]:null,h=c.cache;if(!(e&&!f&&typeof b==="string"&&d===A)){if(e)f||(a[c.expando]=f=++c.uuid);else h=a;if(typeof b==="object")if(e)h[f]=
-c.extend(h[f],b);else c.extend(h,b);else if(e&&!h[f])h[f]={};a=e?h[f]:h;if(d!==A)a[b]=d;return typeof b==="string"?a[b]:a}}},removeData:function(a,b){if(c.acceptData(a)){a=a==E?pa:a;var d=a.nodeType,e=d?a[c.expando]:a,f=c.cache,h=d?f[e]:e;if(b){if(h){delete h[b];d&&c.isEmptyObject(h)&&c.removeData(a)}}else if(d&&c.support.deleteExpando)delete a[c.expando];else if(a.removeAttribute)a.removeAttribute(c.expando);else if(d)delete f[e];else for(var k in a)delete a[k]}},acceptData:function(a){if(a.nodeName){var b=
-c.noData[a.nodeName.toLowerCase()];if(b)return!(b===true||a.getAttribute("classid")!==b)}return true}});c.fn.extend({data:function(a,b){if(typeof a==="undefined")return this.length?c.data(this[0]):null;else if(typeof a==="object")return this.each(function(){c.data(this,a)});var d=a.split(".");d[1]=d[1]?"."+d[1]:"";if(b===A){var e=this.triggerHandler("getData"+d[1]+"!",[d[0]]);if(e===A&&this.length){e=c.data(this[0],a);if(e===A&&this[0].nodeType===1){e=this[0].getAttribute("data-"+a);if(typeof e===
-"string")try{e=e==="true"?true:e==="false"?false:e==="null"?null:!c.isNaN(e)?parseFloat(e):Oa.test(e)?c.parseJSON(e):e}catch(f){}else e=A}}return e===A&&d[1]?this.data(d[0]):e}else return this.each(function(){var h=c(this),k=[d[0],b];h.triggerHandler("setData"+d[1]+"!",k);c.data(this,a,b);h.triggerHandler("changeData"+d[1]+"!",k)})},removeData:function(a){return this.each(function(){c.removeData(this,a)})}});c.extend({queue:function(a,b,d){if(a){b=(b||"fx")+"queue";var e=c.data(a,b);if(!d)return e||
-[];if(!e||c.isArray(d))e=c.data(a,b,c.makeArray(d));else e.push(d);return e}},dequeue:function(a,b){b=b||"fx";var d=c.queue(a,b),e=d.shift();if(e==="inprogress")e=d.shift();if(e){b==="fx"&&d.unshift("inprogress");e.call(a,function(){c.dequeue(a,b)})}}});c.fn.extend({queue:function(a,b){if(typeof a!=="string"){b=a;a="fx"}if(b===A)return c.queue(this[0],a);return this.each(function(){var d=c.queue(this,a,b);a==="fx"&&d[0]!=="inprogress"&&c.dequeue(this,a)})},dequeue:function(a){return this.each(function(){c.dequeue(this,
-a)})},delay:function(a,b){a=c.fx?c.fx.speeds[a]||a:a;b=b||"fx";return this.queue(b,function(){var d=this;setTimeout(function(){c.dequeue(d,b)},a)})},clearQueue:function(a){return this.queue(a||"fx",[])}});var qa=/[\n\t]/g,ga=/\s+/,Pa=/\r/g,Qa=/^(?:href|src|style)$/,Ra=/^(?:button|input)$/i,Sa=/^(?:button|input|object|select|textarea)$/i,Ta=/^a(?:rea)?$/i,ra=/^(?:radio|checkbox)$/i;c.fn.extend({attr:function(a,b){return c.access(this,a,b,true,c.attr)},removeAttr:function(a){return this.each(function(){c.attr(this,
-a,"");this.nodeType===1&&this.removeAttribute(a)})},addClass:function(a){if(c.isFunction(a))return this.each(function(s){var v=c(this);v.addClass(a.call(this,s,v.attr("class")))});if(a&&typeof a==="string")for(var b=(a||"").split(ga),d=0,e=this.length;d-1)return true;return false},
-val:function(a){if(!arguments.length){var b=this[0];if(b){if(c.nodeName(b,"option")){var d=b.attributes.value;return!d||d.specified?b.value:b.text}if(c.nodeName(b,"select")){var e=b.selectedIndex;d=[];var f=b.options;b=b.type==="select-one";if(e<0)return null;var h=b?e:0;for(e=b?e+1:f.length;h=0;else if(c.nodeName(this,"select")){var B=c.makeArray(v);c("option",this).each(function(){this.selected=
-c.inArray(c(this).val(),B)>=0});if(!B.length)this.selectedIndex=-1}else this.value=v}})}});c.extend({attrFn:{val:true,css:true,html:true,text:true,data:true,width:true,height:true,offset:true},attr:function(a,b,d,e){if(!a||a.nodeType===3||a.nodeType===8)return A;if(e&&b in c.attrFn)return c(a)[b](d);e=a.nodeType!==1||!c.isXMLDoc(a);var f=d!==A;b=e&&c.props[b]||b;if(a.nodeType===1){var h=Qa.test(b);if((b in a||a[b]!==A)&&e&&!h){if(f){b==="type"&&Ra.test(a.nodeName)&&a.parentNode&&c.error("type property can't be changed");
-if(d===null)a.nodeType===1&&a.removeAttribute(b);else a[b]=d}if(c.nodeName(a,"form")&&a.getAttributeNode(b))return a.getAttributeNode(b).nodeValue;if(b==="tabIndex")return(b=a.getAttributeNode("tabIndex"))&&b.specified?b.value:Sa.test(a.nodeName)||Ta.test(a.nodeName)&&a.href?0:A;return a[b]}if(!c.support.style&&e&&b==="style"){if(f)a.style.cssText=""+d;return a.style.cssText}f&&a.setAttribute(b,""+d);if(!a.attributes[b]&&a.hasAttribute&&!a.hasAttribute(b))return A;a=!c.support.hrefNormalized&&e&&
-h?a.getAttribute(b,2):a.getAttribute(b);return a===null?A:a}}});var X=/\.(.*)$/,ha=/^(?:textarea|input|select)$/i,Ha=/\./g,Ia=/ /g,Ua=/[^\w\s.|`]/g,Va=function(a){return a.replace(Ua,"\\$&")},sa={focusin:0,focusout:0};c.event={add:function(a,b,d,e){if(!(a.nodeType===3||a.nodeType===8)){if(c.isWindow(a)&&a!==E&&!a.frameElement)a=E;if(d===false)d=U;var f,h;if(d.handler){f=d;d=f.handler}if(!d.guid)d.guid=c.guid++;if(h=c.data(a)){var k=a.nodeType?"events":"__events__",l=h[k],n=h.handle;if(typeof l===
-"function"){n=l.handle;l=l.events}else if(!l){a.nodeType||(h[k]=h=function(){});h.events=l={}}if(!n)h.handle=n=function(){return typeof c!=="undefined"&&!c.event.triggered?c.event.handle.apply(n.elem,arguments):A};n.elem=a;b=b.split(" ");for(var s=0,v;k=b[s++];){h=f?c.extend({},f):{handler:d,data:e};if(k.indexOf(".")>-1){v=k.split(".");k=v.shift();h.namespace=v.slice(0).sort().join(".")}else{v=[];h.namespace=""}h.type=k;if(!h.guid)h.guid=d.guid;var B=l[k],D=c.event.special[k]||{};if(!B){B=l[k]=[];
-if(!D.setup||D.setup.call(a,e,v,n)===false)if(a.addEventListener)a.addEventListener(k,n,false);else a.attachEvent&&a.attachEvent("on"+k,n)}if(D.add){D.add.call(a,h);if(!h.handler.guid)h.handler.guid=d.guid}B.push(h);c.event.global[k]=true}a=null}}},global:{},remove:function(a,b,d,e){if(!(a.nodeType===3||a.nodeType===8)){if(d===false)d=U;var f,h,k=0,l,n,s,v,B,D,H=a.nodeType?"events":"__events__",w=c.data(a),G=w&&w[H];if(w&&G){if(typeof G==="function"){w=G;G=G.events}if(b&&b.type){d=b.handler;b=b.type}if(!b||
-typeof b==="string"&&b.charAt(0)==="."){b=b||"";for(f in G)c.event.remove(a,f+b)}else{for(b=b.split(" ");f=b[k++];){v=f;l=f.indexOf(".")<0;n=[];if(!l){n=f.split(".");f=n.shift();s=RegExp("(^|\\.)"+c.map(n.slice(0).sort(),Va).join("\\.(?:.*\\.)?")+"(\\.|$)")}if(B=G[f])if(d){v=c.event.special[f]||{};for(h=e||0;h=0){a.type=
-f=f.slice(0,-1);a.exclusive=true}if(!d){a.stopPropagation();c.event.global[f]&&c.each(c.cache,function(){this.events&&this.events[f]&&c.event.trigger(a,b,this.handle.elem)})}if(!d||d.nodeType===3||d.nodeType===8)return A;a.result=A;a.target=d;b=c.makeArray(b);b.unshift(a)}a.currentTarget=d;(e=d.nodeType?c.data(d,"handle"):(c.data(d,"__events__")||{}).handle)&&e.apply(d,b);e=d.parentNode||d.ownerDocument;try{if(!(d&&d.nodeName&&c.noData[d.nodeName.toLowerCase()]))if(d["on"+f]&&d["on"+f].apply(d,b)===
-false){a.result=false;a.preventDefault()}}catch(h){}if(!a.isPropagationStopped()&&e)c.event.trigger(a,b,e,true);else if(!a.isDefaultPrevented()){e=a.target;var k,l=f.replace(X,""),n=c.nodeName(e,"a")&&l==="click",s=c.event.special[l]||{};if((!s._default||s._default.call(d,a)===false)&&!n&&!(e&&e.nodeName&&c.noData[e.nodeName.toLowerCase()])){try{if(e[l]){if(k=e["on"+l])e["on"+l]=null;c.event.triggered=true;e[l]()}}catch(v){}if(k)e["on"+l]=k;c.event.triggered=false}}},handle:function(a){var b,d,e;
-d=[];var f,h=c.makeArray(arguments);a=h[0]=c.event.fix(a||E.event);a.currentTarget=this;b=a.type.indexOf(".")<0&&!a.exclusive;if(!b){e=a.type.split(".");a.type=e.shift();d=e.slice(0).sort();e=RegExp("(^|\\.)"+d.join("\\.(?:.*\\.)?")+"(\\.|$)")}a.namespace=a.namespace||d.join(".");f=c.data(this,this.nodeType?"events":"__events__");if(typeof f==="function")f=f.events;d=(f||{})[a.type];if(f&&d){d=d.slice(0);f=0;for(var k=d.length;f-1?c.map(a.options,function(e){return e.selected}).join("-"):"";else if(a.nodeName.toLowerCase()==="select")d=a.selectedIndex;return d},Z=function(a,b){var d=a.target,e,f;if(!(!ha.test(d.nodeName)||d.readOnly)){e=c.data(d,"_change_data");f=va(d);if(a.type!=="focusout"||d.type!=="radio")c.data(d,"_change_data",f);if(!(e===A||f===e))if(e!=null||f){a.type="change";a.liveFired=
-A;return c.event.trigger(a,b,d)}}};c.event.special.change={filters:{focusout:Z,beforedeactivate:Z,click:function(a){var b=a.target,d=b.type;if(d==="radio"||d==="checkbox"||b.nodeName.toLowerCase()==="select")return Z.call(this,a)},keydown:function(a){var b=a.target,d=b.type;if(a.keyCode===13&&b.nodeName.toLowerCase()!=="textarea"||a.keyCode===32&&(d==="checkbox"||d==="radio")||d==="select-multiple")return Z.call(this,a)},beforeactivate:function(a){a=a.target;c.data(a,"_change_data",va(a))}},setup:function(){if(this.type===
-"file")return false;for(var a in V)c.event.add(this,a+".specialChange",V[a]);return ha.test(this.nodeName)},teardown:function(){c.event.remove(this,".specialChange");return ha.test(this.nodeName)}};V=c.event.special.change.filters;V.focus=V.beforeactivate}u.addEventListener&&c.each({focus:"focusin",blur:"focusout"},function(a,b){function d(e){e=c.event.fix(e);e.type=b;return c.event.trigger(e,null,e.target)}c.event.special[b]={setup:function(){sa[b]++===0&&u.addEventListener(a,d,true)},teardown:function(){--sa[b]===
-0&&u.removeEventListener(a,d,true)}}});c.each(["bind","one"],function(a,b){c.fn[b]=function(d,e,f){if(typeof d==="object"){for(var h in d)this[b](h,e,d[h],f);return this}if(c.isFunction(e)||e===false){f=e;e=A}var k=b==="one"?c.proxy(f,function(n){c(this).unbind(n,k);return f.apply(this,arguments)}):f;if(d==="unload"&&b!=="one")this.one(d,e,f);else{h=0;for(var l=this.length;h0?this.bind(b,d,e):this.trigger(b)};if(c.attrFn)c.attrFn[b]=true});E.attachEvent&&!E.addEventListener&&c(E).bind("unload",function(){for(var a in c.cache)if(c.cache[a].handle)try{c.event.remove(c.cache[a].handle.elem)}catch(b){}});
-(function(){function a(g,j,o,m,p,q){p=0;for(var t=m.length;p0){C=x;break}}x=x[g]}m[p]=C}}}var d=/((?:\((?:\([^()]+\)|[^()]+)+\)|\[(?:\[[^\[\]]*\]|['"][^'"]*['"]|[^\[\]'"]+)+\]|\\.|[^ >+~,(\[\\]+)+|[>+~])(\s*,\s*)?((?:.|\r|\n)*)/g,e=0,f=Object.prototype.toString,h=false,k=true;[0,0].sort(function(){k=false;return 0});var l=function(g,j,o,m){o=o||[];var p=j=j||u;if(j.nodeType!==1&&j.nodeType!==9)return[];if(!g||typeof g!=="string")return o;var q=[],t,x,C,P,N=true,R=l.isXML(j),Q=g,L;do{d.exec("");if(t=d.exec(Q)){Q=t[3];q.push(t[1]);if(t[2]){P=t[3];
-break}}}while(t);if(q.length>1&&s.exec(g))if(q.length===2&&n.relative[q[0]])x=M(q[0]+q[1],j);else for(x=n.relative[q[0]]?[j]:l(q.shift(),j);q.length;){g=q.shift();if(n.relative[g])g+=q.shift();x=M(g,x)}else{if(!m&&q.length>1&&j.nodeType===9&&!R&&n.match.ID.test(q[0])&&!n.match.ID.test(q[q.length-1])){t=l.find(q.shift(),j,R);j=t.expr?l.filter(t.expr,t.set)[0]:t.set[0]}if(j){t=m?{expr:q.pop(),set:D(m)}:l.find(q.pop(),q.length===1&&(q[0]==="~"||q[0]==="+")&&j.parentNode?j.parentNode:j,R);x=t.expr?l.filter(t.expr,
-t.set):t.set;if(q.length>0)C=D(x);else N=false;for(;q.length;){t=L=q.pop();if(n.relative[L])t=q.pop();else L="";if(t==null)t=j;n.relative[L](C,t,R)}}else C=[]}C||(C=x);C||l.error(L||g);if(f.call(C)==="[object Array]")if(N)if(j&&j.nodeType===1)for(g=0;C[g]!=null;g++){if(C[g]&&(C[g]===true||C[g].nodeType===1&&l.contains(j,C[g])))o.push(x[g])}else for(g=0;C[g]!=null;g++)C[g]&&C[g].nodeType===1&&o.push(x[g]);else o.push.apply(o,C);else D(C,o);if(P){l(P,p,o,m);l.uniqueSort(o)}return o};l.uniqueSort=function(g){if(w){h=
-k;g.sort(w);if(h)for(var j=1;j0};l.find=function(g,j,o){var m;if(!g)return[];for(var p=0,q=n.order.length;p":function(g,j){var o=typeof j==="string",m,p=0,q=g.length;if(o&&!/\W/.test(j))for(j=j.toLowerCase();p=0))o||m.push(t);else if(o)j[q]=false;return false},ID:function(g){return g[1].replace(/\\/g,"")},TAG:function(g){return g[1].toLowerCase()},CHILD:function(g){if(g[1]==="nth"){var j=/(-?)(\d*)n((?:\+|-)?\d*)/.exec(g[2]==="even"&&"2n"||g[2]==="odd"&&"2n+1"||!/\D/.test(g[2])&&"0n+"+g[2]||g[2]);g[2]=j[1]+(j[2]||1)-0;g[3]=j[3]-0}g[0]=e++;return g},ATTR:function(g,j,o,
-m,p,q){j=g[1].replace(/\\/g,"");if(!q&&n.attrMap[j])g[1]=n.attrMap[j];if(g[2]==="~=")g[4]=" "+g[4]+" ";return g},PSEUDO:function(g,j,o,m,p){if(g[1]==="not")if((d.exec(g[3])||"").length>1||/^\w/.test(g[3]))g[3]=l(g[3],null,null,j);else{g=l.filter(g[3],j,o,true^p);o||m.push.apply(m,g);return false}else if(n.match.POS.test(g[0])||n.match.CHILD.test(g[0]))return true;return g},POS:function(g){g.unshift(true);return g}},filters:{enabled:function(g){return g.disabled===false&&g.type!=="hidden"},disabled:function(g){return g.disabled===
-true},checked:function(g){return g.checked===true},selected:function(g){return g.selected===true},parent:function(g){return!!g.firstChild},empty:function(g){return!g.firstChild},has:function(g,j,o){return!!l(o[3],g).length},header:function(g){return/h\d/i.test(g.nodeName)},text:function(g){return"text"===g.type},radio:function(g){return"radio"===g.type},checkbox:function(g){return"checkbox"===g.type},file:function(g){return"file"===g.type},password:function(g){return"password"===g.type},submit:function(g){return"submit"===
-g.type},image:function(g){return"image"===g.type},reset:function(g){return"reset"===g.type},button:function(g){return"button"===g.type||g.nodeName.toLowerCase()==="button"},input:function(g){return/input|select|textarea|button/i.test(g.nodeName)}},setFilters:{first:function(g,j){return j===0},last:function(g,j,o,m){return j===m.length-1},even:function(g,j){return j%2===0},odd:function(g,j){return j%2===1},lt:function(g,j,o){return jo[3]-0},nth:function(g,j,o){return o[3]-
-0===j},eq:function(g,j,o){return o[3]-0===j}},filter:{PSEUDO:function(g,j,o,m){var p=j[1],q=n.filters[p];if(q)return q(g,o,j,m);else if(p==="contains")return(g.textContent||g.innerText||l.getText([g])||"").indexOf(j[3])>=0;else if(p==="not"){j=j[3];o=0;for(m=j.length;o=0}},ID:function(g,j){return g.nodeType===1&&g.getAttribute("id")===j},TAG:function(g,j){return j==="*"&&g.nodeType===1||g.nodeName.toLowerCase()===
-j},CLASS:function(g,j){return(" "+(g.className||g.getAttribute("class"))+" ").indexOf(j)>-1},ATTR:function(g,j){var o=j[1];o=n.attrHandle[o]?n.attrHandle[o](g):g[o]!=null?g[o]:g.getAttribute(o);var m=o+"",p=j[2],q=j[4];return o==null?p==="!=":p==="="?m===q:p==="*="?m.indexOf(q)>=0:p==="~="?(" "+m+" ").indexOf(q)>=0:!q?m&&o!==false:p==="!="?m!==q:p==="^="?m.indexOf(q)===0:p==="$="?m.substr(m.length-q.length)===q:p==="|="?m===q||m.substr(0,q.length+1)===q+"-":false},POS:function(g,j,o,m){var p=n.setFilters[j[2]];
-if(p)return p(g,o,j,m)}}},s=n.match.POS,v=function(g,j){return"\\"+(j-0+1)},B;for(B in n.match){n.match[B]=RegExp(n.match[B].source+/(?![^\[]*\])(?![^\(]*\))/.source);n.leftMatch[B]=RegExp(/(^(?:.|\r|\n)*?)/.source+n.match[B].source.replace(/\\(\d+)/g,v))}var D=function(g,j){g=Array.prototype.slice.call(g,0);if(j){j.push.apply(j,g);return j}return g};try{Array.prototype.slice.call(u.documentElement.childNodes,0)}catch(H){D=function(g,j){var o=j||[],m=0;if(f.call(g)==="[object Array]")Array.prototype.push.apply(o,
-g);else if(typeof g.length==="number")for(var p=g.length;m";var o=u.documentElement;o.insertBefore(g,o.firstChild);if(u.getElementById(j)){n.find.ID=function(m,p,q){if(typeof p.getElementById!=="undefined"&&!q)return(p=p.getElementById(m[1]))?p.id===m[1]||typeof p.getAttributeNode!=="undefined"&&p.getAttributeNode("id").nodeValue===m[1]?[p]:A:[]};n.filter.ID=function(m,p){var q=typeof m.getAttributeNode!=="undefined"&&m.getAttributeNode("id");return m.nodeType===1&&q&&q.nodeValue===p}}o.removeChild(g);
-o=g=null})();(function(){var g=u.createElement("div");g.appendChild(u.createComment(""));if(g.getElementsByTagName("*").length>0)n.find.TAG=function(j,o){var m=o.getElementsByTagName(j[1]);if(j[1]==="*"){for(var p=[],q=0;m[q];q++)m[q].nodeType===1&&p.push(m[q]);m=p}return m};g.innerHTML="";if(g.firstChild&&typeof g.firstChild.getAttribute!=="undefined"&&g.firstChild.getAttribute("href")!=="#")n.attrHandle.href=function(j){return j.getAttribute("href",2)};g=null})();u.querySelectorAll&&
-function(){var g=l,j=u.createElement("div");j.innerHTML="";if(!(j.querySelectorAll&&j.querySelectorAll(".TEST").length===0)){l=function(m,p,q,t){p=p||u;if(!t&&!l.isXML(p))if(p.nodeType===9)try{return D(p.querySelectorAll(m),q)}catch(x){}else if(p.nodeType===1&&p.nodeName.toLowerCase()!=="object"){var C=p.id,P=p.id="__sizzle__";try{return D(p.querySelectorAll("#"+P+" "+m),q)}catch(N){}finally{if(C)p.id=C;else p.removeAttribute("id")}}return g(m,p,q,t)};for(var o in g)l[o]=g[o];
-j=null}}();(function(){var g=u.documentElement,j=g.matchesSelector||g.mozMatchesSelector||g.webkitMatchesSelector||g.msMatchesSelector,o=false;try{j.call(u.documentElement,":sizzle")}catch(m){o=true}if(j)l.matchesSelector=function(p,q){try{if(o||!n.match.PSEUDO.test(q))return j.call(p,q)}catch(t){}return l(q,null,null,[p]).length>0}})();(function(){var g=u.createElement("div");g.innerHTML="
";if(!(!g.getElementsByClassName||g.getElementsByClassName("e").length===
-0)){g.lastChild.className="e";if(g.getElementsByClassName("e").length!==1){n.order.splice(1,0,"CLASS");n.find.CLASS=function(j,o,m){if(typeof o.getElementsByClassName!=="undefined"&&!m)return o.getElementsByClassName(j[1])};g=null}}})();l.contains=u.documentElement.contains?function(g,j){return g!==j&&(g.contains?g.contains(j):true)}:function(g,j){return!!(g.compareDocumentPosition(j)&16)};l.isXML=function(g){return(g=(g?g.ownerDocument||g:0).documentElement)?g.nodeName!=="HTML":false};var M=function(g,
-j){for(var o=[],m="",p,q=j.nodeType?[j]:j;p=n.match.PSEUDO.exec(g);){m+=p[0];g=g.replace(n.match.PSEUDO,"")}g=n.relative[g]?g+"*":g;p=0;for(var t=q.length;p0)for(var h=d;h0},closest:function(a,
-b){var d=[],e,f,h=this[0];if(c.isArray(a)){var k={},l,n=1;if(h&&a.length){e=0;for(f=a.length;e-1:c(h).is(e))d.push({selector:l,elem:h,level:n})}h=h.parentNode;n++}}return d}k=$a.test(a)?c(a,b||this.context):null;e=0;for(f=this.length;e-1:c.find.matchesSelector(h,a)){d.push(h);break}else{h=h.parentNode;if(!h||
-!h.ownerDocument||h===b)break}d=d.length>1?c.unique(d):d;return this.pushStack(d,"closest",a)},index:function(a){if(!a||typeof a==="string")return c.inArray(this[0],a?c(a):this.parent().children());return c.inArray(a.jquery?a[0]:a,this)},add:function(a,b){var d=typeof a==="string"?c(a,b||this.context):c.makeArray(a),e=c.merge(this.get(),d);return this.pushStack(!d[0]||!d[0].parentNode||d[0].parentNode.nodeType===11||!e[0]||!e[0].parentNode||e[0].parentNode.nodeType===11?e:c.unique(e))},andSelf:function(){return this.add(this.prevObject)}});
-c.each({parent:function(a){return(a=a.parentNode)&&a.nodeType!==11?a:null},parents:function(a){return c.dir(a,"parentNode")},parentsUntil:function(a,b,d){return c.dir(a,"parentNode",d)},next:function(a){return c.nth(a,2,"nextSibling")},prev:function(a){return c.nth(a,2,"previousSibling")},nextAll:function(a){return c.dir(a,"nextSibling")},prevAll:function(a){return c.dir(a,"previousSibling")},nextUntil:function(a,b,d){return c.dir(a,"nextSibling",d)},prevUntil:function(a,b,d){return c.dir(a,"previousSibling",
-d)},siblings:function(a){return c.sibling(a.parentNode.firstChild,a)},children:function(a){return c.sibling(a.firstChild)},contents:function(a){return c.nodeName(a,"iframe")?a.contentDocument||a.contentWindow.document:c.makeArray(a.childNodes)}},function(a,b){c.fn[a]=function(d,e){var f=c.map(this,b,d);Wa.test(a)||(e=d);if(e&&typeof e==="string")f=c.filter(e,f);f=this.length>1?c.unique(f):f;if((this.length>1||Ya.test(e))&&Xa.test(a))f=f.reverse();return this.pushStack(f,a,Za.call(arguments).join(","))}});
-c.extend({filter:function(a,b,d){if(d)a=":not("+a+")";return b.length===1?c.find.matchesSelector(b[0],a)?[b[0]]:[]:c.find.matches(a,b)},dir:function(a,b,d){var e=[];for(a=a[b];a&&a.nodeType!==9&&(d===A||a.nodeType!==1||!c(a).is(d));){a.nodeType===1&&e.push(a);a=a[b]}return e},nth:function(a,b,d){b=b||1;for(var e=0;a;a=a[d])if(a.nodeType===1&&++e===b)break;return a},sibling:function(a,b){for(var d=[];a;a=a.nextSibling)a.nodeType===1&&a!==b&&d.push(a);return d}});var xa=/ jQuery\d+="(?:\d+|null)"/g,
-$=/^\s+/,ya=/<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:]+)[^>]*)\/>/ig,za=/<([\w:]+)/,ab=/\s]+\/)>/g,O={option:[1,""],legend:[1,"","
"],thead:[1,"","
"],tr:[2,"","
"],td:[3,"","
"],col:[2,"","
"],
-area:[1,"",""],_default:[0,"",""]};O.optgroup=O.option;O.tbody=O.tfoot=O.colgroup=O.caption=O.thead;O.th=O.td;if(!c.support.htmlSerialize)O._default=[1,"div",""];c.fn.extend({text:function(a){if(c.isFunction(a))return this.each(function(b){var d=c(this);d.text(a.call(this,b,d.text()))});if(typeof a!=="object"&&a!==A)return this.empty().append((this[0]&&this[0].ownerDocument||u).createTextNode(a));return c.text(this)},wrapAll:function(a){if(c.isFunction(a))return this.each(function(d){c(this).wrapAll(a.call(this,
-d))});if(this[0]){var b=c(a,this[0].ownerDocument).eq(0).clone(true);this[0].parentNode&&b.insertBefore(this[0]);b.map(function(){for(var d=this;d.firstChild&&d.firstChild.nodeType===1;)d=d.firstChild;return d}).append(this)}return this},wrapInner:function(a){if(c.isFunction(a))return this.each(function(b){c(this).wrapInner(a.call(this,b))});return this.each(function(){var b=c(this),d=b.contents();d.length?d.wrapAll(a):b.append(a)})},wrap:function(a){return this.each(function(){c(this).wrapAll(a)})},
-unwrap:function(){return this.parent().each(function(){c.nodeName(this,"body")||c(this).replaceWith(this.childNodes)}).end()},append:function(){return this.domManip(arguments,true,function(a){this.nodeType===1&&this.appendChild(a)})},prepend:function(){return this.domManip(arguments,true,function(a){this.nodeType===1&&this.insertBefore(a,this.firstChild)})},before:function(){if(this[0]&&this[0].parentNode)return this.domManip(arguments,false,function(b){this.parentNode.insertBefore(b,this)});else if(arguments.length){var a=
-c(arguments[0]);a.push.apply(a,this.toArray());return this.pushStack(a,"before",arguments)}},after:function(){if(this[0]&&this[0].parentNode)return this.domManip(arguments,false,function(b){this.parentNode.insertBefore(b,this.nextSibling)});else if(arguments.length){var a=this.pushStack(this,"after",arguments);a.push.apply(a,c(arguments[0]).toArray());return a}},remove:function(a,b){for(var d=0,e;(e=this[d])!=null;d++)if(!a||c.filter(a,[e]).length){if(!b&&e.nodeType===1){c.cleanData(e.getElementsByTagName("*"));
-c.cleanData([e])}e.parentNode&&e.parentNode.removeChild(e)}return this},empty:function(){for(var a=0,b;(b=this[a])!=null;a++)for(b.nodeType===1&&c.cleanData(b.getElementsByTagName("*"));b.firstChild;)b.removeChild(b.firstChild);return this},clone:function(a){var b=this.map(function(){if(!c.support.noCloneEvent&&!c.isXMLDoc(this)){var d=this.outerHTML,e=this.ownerDocument;if(!d){d=e.createElement("div");d.appendChild(this.cloneNode(true));d=d.innerHTML}return c.clean([d.replace(xa,"").replace(cb,'="$1">').replace($,
-"")],e)[0]}else return this.cloneNode(true)});if(a===true){la(this,b);la(this.find("*"),b.find("*"))}return b},html:function(a){if(a===A)return this[0]&&this[0].nodeType===1?this[0].innerHTML.replace(xa,""):null;else if(typeof a==="string"&&!Aa.test(a)&&(c.support.leadingWhitespace||!$.test(a))&&!O[(za.exec(a)||["",""])[1].toLowerCase()]){a=a.replace(ya,"<$1>");try{for(var b=0,d=this.length;b0||e.cacheable||this.length>1?l.cloneNode(true):l)}k.length&&c.each(k,Ka)}return this}});c.buildFragment=function(a,b,d){var e,f,h;b=b&&b[0]?b[0].ownerDocument||b[0]:u;if(a.length===1&&typeof a[0]==="string"&&a[0].length<512&&b===u&&!Aa.test(a[0])&&(c.support.checkClone||
-!Ba.test(a[0]))){f=true;if(h=c.fragments[a[0]])if(h!==1)e=h}if(!e){e=b.createDocumentFragment();c.clean(a,b,e,d)}if(f)c.fragments[a[0]]=h?e:1;return{fragment:e,cacheable:f}};c.fragments={};c.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(a,b){c.fn[a]=function(d){var e=[];d=c(d);var f=this.length===1&&this[0].parentNode;if(f&&f.nodeType===11&&f.childNodes.length===1&&d.length===1){d[b](this[0]);return this}else{f=0;for(var h=
-d.length;f0?this.clone(true):this).get();c(d[f])[b](k);e=e.concat(k)}return this.pushStack(e,a,d.selector)}}});c.extend({clean:function(a,b,d,e){b=b||u;if(typeof b.createElement==="undefined")b=b.ownerDocument||b[0]&&b[0].ownerDocument||u;for(var f=[],h=0,k;(k=a[h])!=null;h++){if(typeof k==="number")k+="";if(k){if(typeof k==="string"&&!bb.test(k))k=b.createTextNode(k);else if(typeof k==="string"){k=k.replace(ya,"<$1>");var l=(za.exec(k)||["",""])[1].toLowerCase(),n=O[l]||O._default,
-s=n[0],v=b.createElement("div");for(v.innerHTML=n[1]+k+n[2];s--;)v=v.lastChild;if(!c.support.tbody){s=ab.test(k);l=l==="table"&&!s?v.firstChild&&v.firstChild.childNodes:n[1]===""&&!s?v.childNodes:[];for(n=l.length-1;n>=0;--n)c.nodeName(l[n],"tbody")&&!l[n].childNodes.length&&l[n].parentNode.removeChild(l[n])}!c.support.leadingWhitespace&&$.test(k)&&v.insertBefore(b.createTextNode($.exec(k)[0]),v.firstChild);k=v.childNodes}if(k.nodeType)f.push(k);else f=c.merge(f,k)}}if(d)for(h=0;f[h];h++)if(e&&
-c.nodeName(f[h],"script")&&(!f[h].type||f[h].type.toLowerCase()==="text/javascript"))e.push(f[h].parentNode?f[h].parentNode.removeChild(f[h]):f[h]);else{f[h].nodeType===1&&f.splice.apply(f,[h+1,0].concat(c.makeArray(f[h].getElementsByTagName("script"))));d.appendChild(f[h])}return f},cleanData:function(a){for(var b,d,e=c.cache,f=c.event.special,h=c.support.deleteExpando,k=0,l;(l=a[k])!=null;k++)if(!(l.nodeName&&c.noData[l.nodeName.toLowerCase()]))if(d=l[c.expando]){if((b=e[d])&&b.events)for(var n in b.events)f[n]?
-c.event.remove(l,n):c.removeEvent(l,n,b.handle);if(h)delete l[c.expando];else l.removeAttribute&&l.removeAttribute(c.expando);delete e[d]}}});var Ca=/alpha\([^)]*\)/i,db=/opacity=([^)]*)/,eb=/-([a-z])/ig,fb=/([A-Z])/g,Da=/^-?\d+(?:px)?$/i,gb=/^-?\d/,hb={position:"absolute",visibility:"hidden",display:"block"},La=["Left","Right"],Ma=["Top","Bottom"],W,ib=u.defaultView&&u.defaultView.getComputedStyle,jb=function(a,b){return b.toUpperCase()};c.fn.css=function(a,b){if(arguments.length===2&&b===A)return this;
-return c.access(this,a,b,true,function(d,e,f){return f!==A?c.style(d,e,f):c.css(d,e)})};c.extend({cssHooks:{opacity:{get:function(a,b){if(b){var d=W(a,"opacity","opacity");return d===""?"1":d}else return a.style.opacity}}},cssNumber:{zIndex:true,fontWeight:true,opacity:true,zoom:true,lineHeight:true},cssProps:{"float":c.support.cssFloat?"cssFloat":"styleFloat"},style:function(a,b,d,e){if(!(!a||a.nodeType===3||a.nodeType===8||!a.style)){var f,h=c.camelCase(b),k=a.style,l=c.cssHooks[h];b=c.cssProps[h]||
-h;if(d!==A){if(!(typeof d==="number"&&isNaN(d)||d==null)){if(typeof d==="number"&&!c.cssNumber[h])d+="px";if(!l||!("set"in l)||(d=l.set(a,d))!==A)try{k[b]=d}catch(n){}}}else{if(l&&"get"in l&&(f=l.get(a,false,e))!==A)return f;return k[b]}}},css:function(a,b,d){var e,f=c.camelCase(b),h=c.cssHooks[f];b=c.cssProps[f]||f;if(h&&"get"in h&&(e=h.get(a,true,d))!==A)return e;else if(W)return W(a,b,f)},swap:function(a,b,d){var e={},f;for(f in b){e[f]=a.style[f];a.style[f]=b[f]}d.call(a);for(f in b)a.style[f]=
-e[f]},camelCase:function(a){return a.replace(eb,jb)}});c.curCSS=c.css;c.each(["height","width"],function(a,b){c.cssHooks[b]={get:function(d,e,f){var h;if(e){if(d.offsetWidth!==0)h=ma(d,b,f);else c.swap(d,hb,function(){h=ma(d,b,f)});return h+"px"}},set:function(d,e){if(Da.test(e)){e=parseFloat(e);if(e>=0)return e+"px"}else return e}}});if(!c.support.opacity)c.cssHooks.opacity={get:function(a,b){return db.test((b&&a.currentStyle?a.currentStyle.filter:a.style.filter)||"")?parseFloat(RegExp.$1)/100+"":
-b?"1":""},set:function(a,b){var d=a.style;d.zoom=1;var e=c.isNaN(b)?"":"alpha(opacity="+b*100+")",f=d.filter||"";d.filter=Ca.test(f)?f.replace(Ca,e):d.filter+" "+e}};if(ib)W=function(a,b,d){var e;d=d.replace(fb,"-$1").toLowerCase();if(!(b=a.ownerDocument.defaultView))return A;if(b=b.getComputedStyle(a,null)){e=b.getPropertyValue(d);if(e===""&&!c.contains(a.ownerDocument.documentElement,a))e=c.style(a,d)}return e};else if(u.documentElement.currentStyle)W=function(a,b){var d,e,f=a.currentStyle&&a.currentStyle[b],
-h=a.style;if(!Da.test(f)&&gb.test(f)){d=h.left;e=a.runtimeStyle.left;a.runtimeStyle.left=a.currentStyle.left;h.left=b==="fontSize"?"1em":f||0;f=h.pixelLeft+"px";h.left=d;a.runtimeStyle.left=e}return f};if(c.expr&&c.expr.filters){c.expr.filters.hidden=function(a){var b=a.offsetHeight;return a.offsetWidth===0&&b===0||!c.support.reliableHiddenOffsets&&(a.style.display||c.css(a,"display"))==="none"};c.expr.filters.visible=function(a){return!c.expr.filters.hidden(a)}}var kb=c.now(),lb=/)<[^<]*)*<\/script>/gi,
-mb=/^(?:select|textarea)/i,nb=/^(?:color|date|datetime|email|hidden|month|number|password|range|search|tel|text|time|url|week)$/i,ob=/^(?:GET|HEAD|DELETE)$/,Na=/\[\]$/,T=/\=\?(&|$)/,ia=/\?/,pb=/([?&])_=[^&]*/,qb=/^(\w+:)?\/\/([^\/?#]+)/,rb=/%20/g,sb=/#.*$/,Ea=c.fn.load;c.fn.extend({load:function(a,b,d){if(typeof a!=="string"&&Ea)return Ea.apply(this,arguments);else if(!this.length)return this;var e=a.indexOf(" ");if(e>=0){var f=a.slice(e,a.length);a=a.slice(0,e)}e="GET";if(b)if(c.isFunction(b)){d=
-b;b=null}else if(typeof b==="object"){b=c.param(b,c.ajaxSettings.traditional);e="POST"}var h=this;c.ajax({url:a,type:e,dataType:"html",data:b,complete:function(k,l){if(l==="success"||l==="notmodified")h.html(f?c("").append(k.responseText.replace(lb,"")).find(f):k.responseText);d&&h.each(d,[k.responseText,l,k])}});return this},serialize:function(){return c.param(this.serializeArray())},serializeArray:function(){return this.map(function(){return this.elements?c.makeArray(this.elements):this}).filter(function(){return this.name&&
-!this.disabled&&(this.checked||mb.test(this.nodeName)||nb.test(this.type))}).map(function(a,b){var d=c(this).val();return d==null?null:c.isArray(d)?c.map(d,function(e){return{name:b.name,value:e}}):{name:b.name,value:d}}).get()}});c.each("ajaxStart ajaxStop ajaxComplete ajaxError ajaxSuccess ajaxSend".split(" "),function(a,b){c.fn[b]=function(d){return this.bind(b,d)}});c.extend({get:function(a,b,d,e){if(c.isFunction(b)){e=e||d;d=b;b=null}return c.ajax({type:"GET",url:a,data:b,success:d,dataType:e})},
-getScript:function(a,b){return c.get(a,null,b,"script")},getJSON:function(a,b,d){return c.get(a,b,d,"json")},post:function(a,b,d,e){if(c.isFunction(b)){e=e||d;d=b;b={}}return c.ajax({type:"POST",url:a,data:b,success:d,dataType:e})},ajaxSetup:function(a){c.extend(c.ajaxSettings,a)},ajaxSettings:{url:location.href,global:true,type:"GET",contentType:"application/x-www-form-urlencoded",processData:true,async:true,xhr:function(){return new E.XMLHttpRequest},accepts:{xml:"application/xml, text/xml",html:"text/html",
-script:"text/javascript, application/javascript",json:"application/json, text/javascript",text:"text/plain",_default:"*/*"}},ajax:function(a){var b=c.extend(true,{},c.ajaxSettings,a),d,e,f,h=b.type.toUpperCase(),k=ob.test(h);b.url=b.url.replace(sb,"");b.context=a&&a.context!=null?a.context:b;if(b.data&&b.processData&&typeof b.data!=="string")b.data=c.param(b.data,b.traditional);if(b.dataType==="jsonp"){if(h==="GET")T.test(b.url)||(b.url+=(ia.test(b.url)?"&":"?")+(b.jsonp||"callback")+"=?");else if(!b.data||
-!T.test(b.data))b.data=(b.data?b.data+"&":"")+(b.jsonp||"callback")+"=?";b.dataType="json"}if(b.dataType==="json"&&(b.data&&T.test(b.data)||T.test(b.url))){d=b.jsonpCallback||"jsonp"+kb++;if(b.data)b.data=(b.data+"").replace(T,"="+d+"$1");b.url=b.url.replace(T,"="+d+"$1");b.dataType="script";var l=E[d];E[d]=function(m){f=m;c.handleSuccess(b,w,e,f);c.handleComplete(b,w,e,f);if(c.isFunction(l))l(m);else{E[d]=A;try{delete E[d]}catch(p){}}v&&v.removeChild(B)}}if(b.dataType==="script"&&b.cache===null)b.cache=
-false;if(b.cache===false&&h==="GET"){var n=c.now(),s=b.url.replace(pb,"$1_="+n);b.url=s+(s===b.url?(ia.test(b.url)?"&":"?")+"_="+n:"")}if(b.data&&h==="GET")b.url+=(ia.test(b.url)?"&":"?")+b.data;b.global&&c.active++===0&&c.event.trigger("ajaxStart");n=(n=qb.exec(b.url))&&(n[1]&&n[1]!==location.protocol||n[2]!==location.host);if(b.dataType==="script"&&h==="GET"&&n){var v=u.getElementsByTagName("head")[0]||u.documentElement,B=u.createElement("script");if(b.scriptCharset)B.charset=b.scriptCharset;B.src=
-b.url;if(!d){var D=false;B.onload=B.onreadystatechange=function(){if(!D&&(!this.readyState||this.readyState==="loaded"||this.readyState==="complete")){D=true;c.handleSuccess(b,w,e,f);c.handleComplete(b,w,e,f);B.onload=B.onreadystatechange=null;v&&B.parentNode&&v.removeChild(B)}}}v.insertBefore(B,v.firstChild);return A}var H=false,w=b.xhr();if(w){b.username?w.open(h,b.url,b.async,b.username,b.password):w.open(h,b.url,b.async);try{if(b.data!=null&&!k||a&&a.contentType)w.setRequestHeader("Content-Type",
-b.contentType);if(b.ifModified){c.lastModified[b.url]&&w.setRequestHeader("If-Modified-Since",c.lastModified[b.url]);c.etag[b.url]&&w.setRequestHeader("If-None-Match",c.etag[b.url])}n||w.setRequestHeader("X-Requested-With","XMLHttpRequest");w.setRequestHeader("Accept",b.dataType&&b.accepts[b.dataType]?b.accepts[b.dataType]+", */*; q=0.01":b.accepts._default)}catch(G){}if(b.beforeSend&&b.beforeSend.call(b.context,w,b)===false){b.global&&c.active--===1&&c.event.trigger("ajaxStop");w.abort();return false}b.global&&
-c.triggerGlobal(b,"ajaxSend",[w,b]);var M=w.onreadystatechange=function(m){if(!w||w.readyState===0||m==="abort"){H||c.handleComplete(b,w,e,f);H=true;if(w)w.onreadystatechange=c.noop}else if(!H&&w&&(w.readyState===4||m==="timeout")){H=true;w.onreadystatechange=c.noop;e=m==="timeout"?"timeout":!c.httpSuccess(w)?"error":b.ifModified&&c.httpNotModified(w,b.url)?"notmodified":"success";var p;if(e==="success")try{f=c.httpData(w,b.dataType,b)}catch(q){e="parsererror";p=q}if(e==="success"||e==="notmodified")d||
-c.handleSuccess(b,w,e,f);else c.handleError(b,w,e,p);d||c.handleComplete(b,w,e,f);m==="timeout"&&w.abort();if(b.async)w=null}};try{var g=w.abort;w.abort=function(){w&&g.call&&g.call(w);M("abort")}}catch(j){}b.async&&b.timeout>0&&setTimeout(function(){w&&!H&&M("timeout")},b.timeout);try{w.send(k||b.data==null?null:b.data)}catch(o){c.handleError(b,w,null,o);c.handleComplete(b,w,e,f)}b.async||M();return w}},param:function(a,b){var d=[],e=function(h,k){k=c.isFunction(k)?k():k;d[d.length]=encodeURIComponent(h)+
-"="+encodeURIComponent(k)};if(b===A)b=c.ajaxSettings.traditional;if(c.isArray(a)||a.jquery)c.each(a,function(){e(this.name,this.value)});else for(var f in a)ca(f,a[f],b,e);return d.join("&").replace(rb,"+")}});c.extend({active:0,lastModified:{},etag:{},handleError:function(a,b,d,e){a.error&&a.error.call(a.context,b,d,e);a.global&&c.triggerGlobal(a,"ajaxError",[b,a,e])},handleSuccess:function(a,b,d,e){a.success&&a.success.call(a.context,e,d,b);a.global&&c.triggerGlobal(a,"ajaxSuccess",[b,a])},handleComplete:function(a,
-b,d){a.complete&&a.complete.call(a.context,b,d);a.global&&c.triggerGlobal(a,"ajaxComplete",[b,a]);a.global&&c.active--===1&&c.event.trigger("ajaxStop")},triggerGlobal:function(a,b,d){(a.context&&a.context.url==null?c(a.context):c.event).trigger(b,d)},httpSuccess:function(a){try{return!a.status&&location.protocol==="file:"||a.status>=200&&a.status<300||a.status===304||a.status===1223}catch(b){}return false},httpNotModified:function(a,b){var d=a.getResponseHeader("Last-Modified"),e=a.getResponseHeader("Etag");
-if(d)c.lastModified[b]=d;if(e)c.etag[b]=e;return a.status===304},httpData:function(a,b,d){var e=a.getResponseHeader("content-type")||"",f=b==="xml"||!b&&e.indexOf("xml")>=0;a=f?a.responseXML:a.responseText;f&&a.documentElement.nodeName==="parsererror"&&c.error("parsererror");if(d&&d.dataFilter)a=d.dataFilter(a,b);if(typeof a==="string")if(b==="json"||!b&&e.indexOf("json")>=0)a=c.parseJSON(a);else if(b==="script"||!b&&e.indexOf("javascript")>=0)c.globalEval(a);return a}});if(E.ActiveXObject)c.ajaxSettings.xhr=
-function(){if(E.location.protocol!=="file:")try{return new E.XMLHttpRequest}catch(a){}try{return new E.ActiveXObject("Microsoft.XMLHTTP")}catch(b){}};c.support.ajax=!!c.ajaxSettings.xhr();var da={},tb=/^(?:toggle|show|hide)$/,ub=/^([+\-]=)?([\d+.\-]+)(.*)$/,aa,na=[["height","marginTop","marginBottom","paddingTop","paddingBottom"],["width","marginLeft","marginRight","paddingLeft","paddingRight"],["opacity"]];c.fn.extend({show:function(a,b,d){if(a||a===0)return this.animate(S("show",3),a,b,d);else{a=
-0;for(b=this.length;a=0;e--)if(d[e].elem===this){b&&d[e](true);d.splice(e,1)}});b||this.dequeue();return this}});c.each({slideDown:S("show",1),slideUp:S("hide",1),slideToggle:S("toggle",1),fadeIn:{opacity:"show"},fadeOut:{opacity:"hide"}},function(a,b){c.fn[a]=function(d,e,f){return this.animate(b,
-d,e,f)}});c.extend({speed:function(a,b,d){var e=a&&typeof a==="object"?c.extend({},a):{complete:d||!d&&b||c.isFunction(a)&&a,duration:a,easing:d&&b||b&&!c.isFunction(b)&&b};e.duration=c.fx.off?0:typeof e.duration==="number"?e.duration:e.duration in c.fx.speeds?c.fx.speeds[e.duration]:c.fx.speeds._default;e.old=e.complete;e.complete=function(){e.queue!==false&&c(this).dequeue();c.isFunction(e.old)&&e.old.call(this)};return e},easing:{linear:function(a,b,d,e){return d+e*a},swing:function(a,b,d,e){return(-Math.cos(a*
-Math.PI)/2+0.5)*e+d}},timers:[],fx:function(a,b,d){this.options=b;this.elem=a;this.prop=d;if(!b.orig)b.orig={}}});c.fx.prototype={update:function(){this.options.step&&this.options.step.call(this.elem,this.now,this);(c.fx.step[this.prop]||c.fx.step._default)(this)},cur:function(){if(this.elem[this.prop]!=null&&(!this.elem.style||this.elem.style[this.prop]==null))return this.elem[this.prop];var a=parseFloat(c.css(this.elem,this.prop));return a&&a>-1E4?a:0},custom:function(a,b,d){function e(h){return f.step(h)}
-this.startTime=c.now();this.start=a;this.end=b;this.unit=d||this.unit||"px";this.now=this.start;this.pos=this.state=0;var f=this;a=c.fx;e.elem=this.elem;if(e()&&c.timers.push(e)&&!aa)aa=setInterval(a.tick,a.interval)},show:function(){this.options.orig[this.prop]=c.style(this.elem,this.prop);this.options.show=true;this.custom(this.prop==="width"||this.prop==="height"?1:0,this.cur());c(this.elem).show()},hide:function(){this.options.orig[this.prop]=c.style(this.elem,this.prop);this.options.hide=true;
-this.custom(this.cur(),0)},step:function(a){var b=c.now(),d=true;if(a||b>=this.options.duration+this.startTime){this.now=this.end;this.pos=this.state=1;this.update();this.options.curAnim[this.prop]=true;for(var e in this.options.curAnim)if(this.options.curAnim[e]!==true)d=false;if(d){if(this.options.overflow!=null&&!c.support.shrinkWrapBlocks){var f=this.elem,h=this.options;c.each(["","X","Y"],function(l,n){f.style["overflow"+n]=h.overflow[l]})}this.options.hide&&c(this.elem).hide();if(this.options.hide||
-this.options.show)for(var k in this.options.curAnim)c.style(this.elem,k,this.options.orig[k]);this.options.complete.call(this.elem)}return false}else{a=b-this.startTime;this.state=a/this.options.duration;b=this.options.easing||(c.easing.swing?"swing":"linear");this.pos=c.easing[this.options.specialEasing&&this.options.specialEasing[this.prop]||b](this.state,a,0,1,this.options.duration);this.now=this.start+(this.end-this.start)*this.pos;this.update()}return true}};c.extend(c.fx,{tick:function(){for(var a=
-c.timers,b=0;b-1;e={};var s={};if(n)s=f.position();k=n?s.top:parseInt(k,10)||0;l=n?s.left:parseInt(l,10)||0;if(c.isFunction(b))b=b.call(a,d,h);if(b.top!=null)e.top=b.top-h.top+k;if(b.left!=null)e.left=b.left-h.left+l;"using"in b?b.using.call(a,
-e):f.css(e)}};c.fn.extend({position:function(){if(!this[0])return null;var a=this[0],b=this.offsetParent(),d=this.offset(),e=Fa.test(b[0].nodeName)?{top:0,left:0}:b.offset();d.top-=parseFloat(c.css(a,"marginTop"))||0;d.left-=parseFloat(c.css(a,"marginLeft"))||0;e.top+=parseFloat(c.css(b[0],"borderTopWidth"))||0;e.left+=parseFloat(c.css(b[0],"borderLeftWidth"))||0;return{top:d.top-e.top,left:d.left-e.left}},offsetParent:function(){return this.map(function(){for(var a=this.offsetParent||u.body;a&&!Fa.test(a.nodeName)&&
-c.css(a,"position")==="static";)a=a.offsetParent;return a})}});c.each(["Left","Top"],function(a,b){var d="scroll"+b;c.fn[d]=function(e){var f=this[0],h;if(!f)return null;if(e!==A)return this.each(function(){if(h=ea(this))h.scrollTo(!a?e:c(h).scrollLeft(),a?e:c(h).scrollTop());else this[d]=e});else return(h=ea(f))?"pageXOffset"in h?h[a?"pageYOffset":"pageXOffset"]:c.support.boxModel&&h.document.documentElement[d]||h.document.body[d]:f[d]}});c.each(["Height","Width"],function(a,b){var d=b.toLowerCase();
-c.fn["inner"+b]=function(){return this[0]?parseFloat(c.css(this[0],d,"padding")):null};c.fn["outer"+b]=function(e){return this[0]?parseFloat(c.css(this[0],d,e?"margin":"border")):null};c.fn[d]=function(e){var f=this[0];if(!f)return e==null?null:this;if(c.isFunction(e))return this.each(function(h){var k=c(this);k[d](e.call(this,h,k[d]()))});return c.isWindow(f)?f.document.compatMode==="CSS1Compat"&&f.document.documentElement["client"+b]||f.document.body["client"+b]:f.nodeType===9?Math.max(f.documentElement["client"+
-b],f.body["scroll"+b],f.documentElement["scroll"+b],f.body["offset"+b],f.documentElement["offset"+b]):e===A?parseFloat(c.css(f,d)):this.css(d,typeof e==="string"?e:e+"px")}})})(window);
diff --git a/rpython/rlib/rstrategies/htmlcov1/jquery.tablesorter.min.js b/rpython/rlib/rstrategies/htmlcov1/jquery.tablesorter.min.js
deleted file mode 100644
--- a/rpython/rlib/rstrategies/htmlcov1/jquery.tablesorter.min.js
+++ /dev/null
@@ -1,2 +0,0 @@
-
-(function($){$.extend({tablesorter:new function(){var parsers=[],widgets=[];this.defaults={cssHeader:"header",cssAsc:"headerSortUp",cssDesc:"headerSortDown",sortInitialOrder:"asc",sortMultiSortKey:"shiftKey",sortForce:null,sortAppend:null,textExtraction:"simple",parsers:{},widgets:[],widgetZebra:{css:["even","odd"]},headers:{},widthFixed:false,cancelSelection:true,sortList:[],headerList:[],dateFormat:"us",decimal:'.',debug:false};function benchmark(s,d){log(s+","+(new Date().getTime()-d.getTime())+"ms");}this.benchmark=benchmark;function log(s){if(typeof console!="undefined"&&typeof console.debug!="undefined"){console.log(s);}else{alert(s);}}function buildParserCache(table,$headers){if(table.config.debug){var parsersDebug="";}var rows=table.tBodies[0].rows;if(table.tBodies[0].rows[0]){var list=[],cells=rows[0].cells,l=cells.length;for(var i=0;i1){arr=arr.concat(checkCellColSpan(table,headerArr,row++));}else{if(table.tHead.length==1||(cell.rowSpan>1||!r[row+1])){arr.push(cell);}}}return arr;};function checkHeaderMetadata(cell){if(($.metadata)&&($(cell).metadata().sorter===false)){return true;};return false;}function checkHeaderOptions(table,i){if((table.config.headers[i])&&(table.config.headers[i].sorter===false)){return true;};return false;}function applyWidget(table){var c=table.config.widgets;var l=c.length;for(var i=0;i');$("tr:first td",table.tBodies[0]).each(function(){colgroup.append($('').css('width',$(this).width()));});$(table).prepend(colgroup);};}function updateHeaderSortCount(table,sortList){var c=table.config,l=sortList.length;for(var i=0;ib)?1:0));};function sortTextDesc(a,b){return((ba)?1:0));};function sortNumeric(a,b){return a-b;};function sortNumericDesc(a,b){return b-a;};function getCachedSortType(parsers,i){return parsers[i].type;};this.construct=function(settings){return this.each(function(){if(!this.tHead||!this.tBodies)return;var $this,$document,$headers,cache,config,shiftDown=0,sortOrder;this.config={};config=$.extend(this.config,$.tablesorter.defaults,settings);$this=$(this);$headers=buildHeaders(this);this.config.parsers=buildParserCache(this,$headers);cache=buildCache(this);var sortCSS=[config.cssDesc,config.cssAsc];fixColumnWidth(this);$headers.click(function(e){$this.trigger("sortStart");var totalRows=($this[0].tBodies[0]&&$this[0].tBodies[0].rows.length)||0;if(!this.sortDisabled&&totalRows>0){var $cell=$(this);var i=this.column;this.order=this.count++%2;if(!e[config.sortMultiSortKey]){config.sortList=[];if(config.sortForce!=null){var a=config.sortForce;for(var j=0;j0){$this.trigger("sorton",[config.sortList]);}applyWidget(this);});};this.addParser=function(parser){var l=parsers.length,a=true;for(var i=0;i
-
-
-    
-    
-    
-    
-    Coverage for logger: 96%
-    
-    
-    
-    
-    
-    
-    
-
-
-
-
-    
-        Coverage for logger :
-            96%
-        
-        
-        
-            46 statements  
-            44 run
-            2 missing
-            0 excluded
-            
-        
-    
-
-
-
-    
-Hot-keys on this page
-    
-
-        r
-        m
-        x
-        p   toggle line displays
-    
-
-        j
-        k   next/prev highlighted chunk
-    
-
-        0   (zero) top of page
-    
-
-        1   (one) first highlighted chunk
-    
-    
-
-
-
-    
-        
-            
-            
-        
-    
-1
-2
-3
-4
-5
-6
-7
-8
-9
-10
-11
-12
-13
-14
-15
-16
-17
-18
-19
-20
-21
-22
-23
-24
-25
-26
-27
-28
-29
-30
-31
-32
-33
-34
-35
-36
-37
-38
-39
-40
-41
-42
-43
-44
-45
-46
-47
-48
-49
-50
-51
-52
-53
-54
-55
-56
-57
-58
-                
-            
- 
-class LogEntry(object): 
-    def __init__(self): 
-        self.slots = 0 
-        self.objects = 0 
-        self.element_typenames = {} 
- 
-    def add(self, size, element_typename): 
-        self.slots += size 
-        self.objects += 1 
-        if element_typename: 
-            self.element_typenames[element_typename] = None 
- 
-    def classnames(self): 
-        return self.element_typenames.keys() 
- 
-class Logger(object): 
-    _attrs_ = ["active", "aggregate", "logs"] 
-    _immutable_fields_ = ["active?", "aggregate?", "logs"] 
- 
-    def __init__(self): 
-        self.active = False 
-        self.aggregate = False 
-        self.logs = {} 
- 
-    def activate(self, aggregate=False): 
-        self.active = True 
-        self.aggregate = self.aggregate or aggregate 
- 
-    def log(self, new_strategy, size, cause="", old_strategy="", typename="", element_typename=""): 
-        if self.aggregate: 
-            key = (cause, old_strategy, new_strategy, typename) 
-            if key not in self.logs: 
-                self.logs[key] = LogEntry() 
-            entry = self.logs[key] 
-            entry.add(size, element_typename) 
-        else: 
-            element_typenames = [ element_typename ] if element_typename else [] 
-            self.output(cause, old_strategy, new_strategy, typename, size, 1, element_typenames) 
- 
-    def print_aggregated_log(self): 
-        if not self.aggregate: 
-            return 
-        for key, entry in self.logs.items(): 
-            cause, old_strategy, new_strategy, typename = key 
-            slots, objects, element_typenames = entry.slots, entry.objects, entry.classnames() 
-            self.output(cause, old_strategy, new_strategy, typename, slots, objects, element_typenames) 
- 
-    def output(self, cause, old_strategy, new_strategy, typename, slots, objects, element_typenames): 
-        old_strategy_string = "%s -> " % old_strategy if old_strategy else "" 
-        classname_string = " of %s" % typename if typename else "" 
-        element_string = (" elements: " + " ".join(element_typenames)) if element_typenames else "" 
-        format = (cause, old_strategy_string, new_strategy, classname_string, slots, objects, element_string) 
-        self.do_print("%s (%s%s)%s size %d objects %d%s" % format) 
- 
-    def do_print(self, str): 
-        # Hook to increase testability 
-        print str 
-                
-            
-

Module	statements	missing	coverage
Total	395	18	95%
logger	46	2	96%
rstrategies	349	16	95%

- - - -

- -

import weakref, sys

from rpython.rlib.rstrategies import logger

from rpython.rlib import jit, objectmodel, rerased

from rpython.rlib.objectmodel import specialize

def make_accessors(strategy='strategy', storage='storage'):

"""

Instead of using this generator, the methods can be implemented manually.

A third way is to overwrite the getter/setter methods in StrategyFactory.

"""

def make_getter(attr):

def getter(self): return getattr(self, attr)

return getter

def make_setter(attr):

def setter(self, val): setattr(self, attr, val)

return setter

classdef = sys._getframe(1).f_locals

classdef['_get_strategy'] = make_getter(strategy)

classdef['_set_strategy'] = make_setter(strategy)

classdef['_get_storage'] = make_getter(storage)

classdef['_set_storage'] = make_setter(storage)

class StrategyMetaclass(type):

"""

A metaclass is required, because we need certain attributes to be special

for every single strategy class.

"""

def __new__(self, name, bases, attrs):

attrs['_is_strategy'] = False

attrs['_is_singleton'] = False

attrs['_specializations'] = []

# Not every strategy uses rerased-pairs, but they won't hurt

erase, unerase = rerased.new_erasing_pair(name)

def get_storage(self, w_self):

erased = self.strategy_factory().get_storage(w_self)

return unerase(erased)

def set_storage(self, w_self, storage):

erased = erase(storage)

self.strategy_factory().set_storage(w_self, erased)

attrs['get_storage'] = get_storage

attrs['set_storage'] = set_storage

return type.__new__(self, name, bases, attrs)

def strategy(generalize=None, singleton=True):

"""

Strategy classes must be decorated with this.

generalize is a list of other strategies, that can be switched to from the decorated strategy.

If the singleton flag is set to False, new strategy instances will be created,

instead of always reusing the singleton object.

"""

def decorator(strategy_class):

# Patch strategy class: Add generalized_strategy_for and mark as strategy class.

if generalize:

@jit.unroll_safe

def generalized_strategy_for(self, value):

# TODO - optimize this method

for strategy in generalize:

if self.strategy_factory().strategy_singleton_instance(strategy)._check_can_handle(value):

return strategy

raise Exception("Could not find generalized strategy for %s coming from %s" % (value, self))

strategy_class.generalized_strategy_for = generalized_strategy_for

for generalized in generalize:

generalized._specializations.append(strategy_class)

strategy_class._is_strategy = True

strategy_class._generalizations = generalize

strategy_class._is_singleton = singleton

return strategy_class

return decorator

class StrategyFactory(object):

_immutable_fields_ = ["strategies[*]", "logger", "strategy_singleton_field"]

factory_instance_counter = 0

def __init__(self, root_class, all_strategy_classes=None):

if all_strategy_classes is None:

all_strategy_classes = self._collect_subclasses(root_class)

self.strategies = []

self.logger = logger.Logger()

# This is to avoid confusion between multiple factories existing simultaneously (e.g. in tests)

self.strategy_singleton_field = "__singleton_%i" % StrategyFactory.factory_instance_counter

StrategyFactory.factory_instance_counter += 1

self._create_strategy_instances(root_class, all_strategy_classes)

def _create_strategy_instances(self, root_class, all_strategy_classes):

for strategy_class in all_strategy_classes:

if strategy_class._is_strategy:

setattr(strategy_class, self.strategy_singleton_field, self.instantiate_strategy(strategy_class))

self.strategies.append(strategy_class)

self._patch_strategy_class(strategy_class, root_class)

self._order_strategies()

# =============================

# API methods

# =============================

def switch_strategy(self, w_self, new_strategy_type, new_element=None):

"""

Switch the strategy of w_self to the new type.

new_element can be given as as hint, purely for logging purposes.

It should be the object that was added to w_self, causing the strategy switch.

"""

old_strategy = self.get_strategy(w_self)

if new_strategy_type._is_singleton:

new_strategy = self.strategy_singleton_instance(new_strategy_type)

else:

size = old_strategy.size(w_self)

new_strategy = self.instantiate_strategy(new_strategy_type, w_self, size)

self.set_strategy(w_self, new_strategy)

old_strategy._convert_storage_to(w_self, new_strategy)

new_strategy.strategy_switched(w_self)

self.log(w_self, new_strategy, old_strategy, new_element)

return new_strategy

def set_initial_strategy(self, w_self, strategy_type, size, elements=None):

"""

Initialize the strategy and storage fields of w_self.

This must be called before switch_strategy or any strategy method can be used.

elements is an optional list of values initially stored in w_self.

If given, then len(elements) == size must hold.

"""

assert self.get_strategy(w_self) is None, "Strategy should not be initialized yet!"

if strategy_type._is_singleton:

strategy = self.strategy_singleton_instance(strategy_type)

else:

strategy = self.instantiate_strategy(strategy_type, w_self, size)

self.set_strategy(w_self, strategy)

strategy._initialize_storage(w_self, size)

element = None

if elements:

strategy.store_all(w_self, elements)

if len(elements) > 0: element = elements[0]

strategy.strategy_switched(w_self)

self.log(w_self, strategy, None, element)

return strategy

@jit.unroll_safe

def strategy_type_for(self, objects):

"""

Return the best-fitting strategy to hold all given objects.

"""

specialized_strategies = len(self.strategies)

can_handle = [True] * specialized_strategies

for obj in objects:

if specialized_strategies <= 1:

break

for i, strategy in enumerate(self.strategies):

if can_handle[i] and not self.strategy_singleton_instance(strategy)._check_can_handle(obj):

can_handle[i] = False

specialized_strategies -= 1

for i, strategy_type in enumerate(self.strategies):

if can_handle[i]:

return strategy_type

raise Exception("Could not find strategy to handle: %s" % objects)

def decorate_strategies(self, transitions):

"""

As an alternative to decorating all strategies with @strategy,

invoke this in the constructor of your StrategyFactory subclass, before

calling __init__. transitions is a dict mapping all strategy classes to

their 'generalize' list parameter (see @strategy decorator).

"""

"NOT_RPYTHON"

for strategy_class, generalized in transitions.items():

strategy(generalized)(strategy_class)

# =============================

# The following methods can be overwritten to customize certain aspects of the factory.

# =============================

def instantiate_strategy(self, strategy_type, w_self=None, initial_size=0):

"""

Return a functional instance of strategy_type.

Overwrite this if you need a non-default constructor.

The two additional parameters should be ignored for singleton-strategies.

"""

return strategy_type()

def log(self, w_self, new_strategy, old_strategy=None, new_element=None):

"""

This can be overwritten into a more appropriate call to self.logger.log

"""

if not self.logger.active: return

new_strategy_str = self.log_string_for_object(new_strategy)

old_strategy_str = self.log_string_for_object(old_strategy)

element_typename = self.log_string_for_object(new_element)

size = new_strategy.size(w_self)

typename = ""

cause = "Switched" if old_strategy else "Created"

self.logger.log(new_strategy_str, size, cause, old_strategy_str, typename, element_typename)

@specialize.call_location()

def log_string_for_object(self, obj):

"""

This can be overwritten instead of the entire log() method.

Keep the specialize-annotation in order to handle different kinds of objects here.

"""

return obj.__class__.__name__ if obj else ""

# These storage accessors are specialized because the storage field is

# populated by erased-objects which seem to be incompatible sometimes.

@specialize.call_location()

def get_storage(self, obj):

return obj._get_storage()

@specialize.call_location()

def set_storage(self, obj, val):

return obj._set_storage(val)

def get_strategy(self, obj):

return obj._get_strategy()

def set_strategy(self, obj, val):

return obj._set_strategy(val)

# =============================

# Internal methods

# =============================

def _patch_strategy_class(self, strategy_class, root_class):

"NOT_RPYTHON"

# Patch root class: Add default handler for visitor

def _convert_storage_from_OTHER(self, w_self, previous_strategy):

self._convert_storage_from(w_self, previous_strategy)

funcname = "_convert_storage_from_" + strategy_class.__name__

_convert_storage_from_OTHER.func_name = funcname

setattr(root_class, funcname, _convert_storage_from_OTHER)

# Patch strategy class: Add polymorphic visitor function

def _convert_storage_to(self, w_self, new_strategy):

getattr(new_strategy, funcname)(w_self, self)

strategy_class._convert_storage_to = _convert_storage_to

def _collect_subclasses(self, cls):

"NOT_RPYTHON"

subclasses = []

for subcls in cls.__subclasses__():

subclasses.append(subcls)

subclasses.extend(self._collect_subclasses(subcls))

return subclasses

def _order_strategies(self):

"NOT_RPYTHON"

def get_generalization_depth(strategy, visited=None):

if visited is None:

visited = set()

if strategy._generalizations:

if strategy in visited:

raise Exception("Cycle in generalization-tree of %s" % strategy)

visited.add(strategy)

depth = 0

for generalization in strategy._generalizations:

other_depth = get_generalization_depth(generalization, set(visited))

depth = max(depth, other_depth)

return depth + 1

else:

return 0

self.strategies.sort(key=get_generalization_depth, reverse=True)

@jit.elidable

def strategy_singleton_instance(self, strategy_class):

return getattr(strategy_class, self.strategy_singleton_field)

def _freeze_(self):

# Instance will be frozen at compile time, making accesses constant.

# The constructor does meta stuff which is not possible after translation.

return True

class AbstractStrategy(object):

"""

== Required:

strategy_factory(self) - Access to StorageFactory

"""

def strategy_switched(self, w_self):

# Overwrite this method for a hook whenever the strategy

# of w_self was switched to self.

pass

# Main Fixedsize API

def store(self, w_self, index0, value):

raise NotImplementedError("Abstract method")

def fetch(self, w_self, index0):

raise NotImplementedError("Abstract method")

def size(self, w_self):

raise NotImplementedError("Abstract method")

# Fixedsize utility methods

def slice(self, w_self, start, end):

return [ self.fetch(w_self, i) for i in range(start, end)]

def fetch_all(self, w_self):

return self.slice(w_self, 0, self.size(w_self))

def store_all(self, w_self, elements):

for i, e in enumerate(elements):

self.store(w_self, i, e)

# Main Varsize API

def insert(self, w_self, index0, list_w):

raise NotImplementedError("Abstract method")

def delete(self, w_self, start, end):

raise NotImplementedError("Abstract method")

# Varsize utility methods

def append(self, w_self, list_w):

self.insert(w_self, self.size(w_self), list_w)

def pop(self, w_self, index0):

e = self.fetch(w_self, index0)

self.delete(w_self, index0, index0+1)

return e

From noreply at buildbot.pypy.org Wed Apr 22 12:21:02 2015 From: noreply at buildbot.pypy.org (anton_gulenko) Date: Wed, 22 Apr 2015 12:21:02 +0200 (CEST) Subject: [pypy-commit] pypy default: Merged file edited online. Message-ID: <20150422102102.291C41C06CD@cobra.cs.uni-duesseldorf.de> Author: Anton Gulenko Branch: Changeset: r76882:f1984a4d5583 Date: 2015-03-12 15:04 +0100 http://bitbucket.org/pypy/pypy/changeset/f1984a4d5583/ Log: Merged file edited online. diff --git a/rpython/doc/rstrategies.rst b/rpython/doc/rstrategies.rst --- a/rpython/doc/rstrategies.rst +++ b/rpython/doc/rstrategies.rst @@ -98,18 +98,18 @@ :: -class AbstractStrategy(AbstractStrategy): - _attrs_ = ['space'] - _immutable_fields_ = ['space'] - __metaclass__ = rstrat.StrategyMetaclass - import_from_mixin(rstrat.AbstractStrategy) - import_from_mixin(rstrategies.SafeIndexingMixin) - - def __init__(self, space): - self.space = space - - def strategy_factory(self): - return self.space.strategy_factory + class AbstractStrategy(AbstractStrategy): + _attrs_ = ['space'] + _immutable_fields_ = ['space'] + __metaclass__ = rstrat.StrategyMetaclass + import_from_mixin(rstrat.AbstractStrategy) + import_from_mixin(rstrategies.SafeIndexingMixin) + + def __init__(self, space): + self.space = space + + def strategy_factory(self): + return self.space.strategy_factory Strategy classes @@ -144,14 +144,14 @@ :: - at rstrat.strategy(generalize=[GenericStrategy]) -class IntegerOrNilStrategy(AbstractStrategy): - import_from_mixin(rstrat.TaggingStrategy) - contained_type = model.W_Integer - def wrap(self, val): return self.space.wrap_int(val) - def unwrap(self, w_val): return self.space.unwrap_int(w_val) - def wrapped_tagged_value(self): return self.space.w_nil - def unwrapped_tagged_value(self): return constants.MAXINT + @rstrat.strategy(generalize=[GenericStrategy]) + class IntegerOrNilStrategy(AbstractStrategy): + import_from_mixin(rstrat.TaggingStrategy) + contained_type = model.W_Integer + def wrap(self, val): return self.space.wrap_int(val) + def unwrap(self, w_val): return self.space.unwrap_int(w_val) + def wrapped_tagged_value(self): return self.space.w_nil + def unwrapped_tagged_value(self): return constants.MAXINT Strategy Factory ---------------- @@ -188,22 +188,22 @@ :: -class StrategyFactory(rstrategies.StrategyFactory): - _attrs_ = ['space'] - _immutable_fields_ = ['space'] - - def __init__(self, space): - self.space = space - rstrat.StrategyFactory.__init__(self, AbstractStrategy) - - def instantiate_strategy(self, strategy_type): - return strategy_type(self.space) - - def strategy_type_for(self, list_w, weak=False): - """ - Helper method for handling weak objects specially - """ - if weak: - return WeakListStrategy + class StrategyFactory(rstrategies.StrategyFactory): + _attrs_ = ['space'] + _immutable_fields_ = ['space'] + + def __init__(self, space): + self.space = space + rstrat.StrategyFactory.__init__(self, AbstractStrategy) + + def instantiate_strategy(self, strategy_type): + return strategy_type(self.space) + + def strategy_type_for(self, list_w, weak=False): + """ + Helper method for handling weak objects specially + """ + if weak: + return WeakListStrategy return rstrategies.StrategyFactory.strategy_type_for(self, list_w) \ No newline at end of file From noreply at buildbot.pypy.org Wed Apr 22 12:21:03 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Wed, 22 Apr 2015 12:21:03 +0200 (CEST) Subject: [pypy-commit] pypy default: Merged in anton_gulenko/pypy (pull request #306) Message-ID: <20150422102103.4FB351C06CD@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: Changeset: r76883:62106262ee6c Date: 2015-04-22 12:20 +0200 http://bitbucket.org/pypy/pypy/changeset/62106262ee6c/ Log: Merged in anton_gulenko/pypy (pull request #306) rstrategies - A library for Storage Strategies diff too long, truncating to 2000 out of 2151 lines diff --git a/rpython/doc/rlib.rst b/rpython/doc/rlib.rst --- a/rpython/doc/rlib.rst +++ b/rpython/doc/rlib.rst @@ -128,6 +128,14 @@ a hierarchy of Address classes, in a typical static-OO-programming style. +rstrategies +----------- + +The :source:`rpython/rlib/rstrategies` module contains a library to implement storage strategies in +RPython VMs. The library is language-independent and extensible. +More details and examples can be found in the :doc:`rstrategies documentation `. + + streamio -------- diff --git a/rpython/doc/rstrategies.rst b/rpython/doc/rstrategies.rst new file mode 100644 --- /dev/null +++ b/rpython/doc/rstrategies.rst @@ -0,0 +1,209 @@ +rstrategies +=========== + +A library to implement storage strategies in VMs based on the RPython +toolchain. rstrategies can be used in VMs for any language or language +family. + +This library has been developed as part of a Masters Thesis by `Anton +Gulenko `__. + +The original paper describing the optimization "Storage Strategies for +collections in dynamically typed languages" by C.F. Bolz, L. Diekmann +and L. Tratt can be found +`here `__. + +So far, this library has been adpoted by 3 VMs: +`RSqueak `__, +`Topaz `__ (`Forked +here `__) and +`Pycket `__ (`Forked +here `__). + +Concept +------- + +Collections are often used homogeneously, i.e. they contain only objects +of the same type. Primitive numeric types like ints or floats are +especially interesting for optimization. These cases can be optimized by +storing the unboxed data of these objects in consecutive memory. This is +done by letting a special "strategy" object handle the entire storage of +a collection. The collection object holds two separate references: one +to its strategy and one to its storage. Every operation on the +collection is delegated to the strategy, which accesses the storage when +needed. The strategy can be switched to a more suitable one, which might +require converting the storage array. + +Usage +~~~~~ + +The following are the steps needed to integrated rstrategies in an +RPython VM. Because of the special nature of this library it is not +enough to simply call some API methods; the library must be integrated +within existing VM classes using a metaclass, mixins and other +meta-programming techniques. + +The sequence of steps described here is something like a "setup +walkthrough", and might be a bit abstract. To see a concrete example, +look at +`SingletonStorageStrategy `__, +`StrategyFactory `__ +and +`W\_PointersObject `__ +from the `RSqueak VM `__. The +code is also well commented. + +Basics +------- + +Currently the rstrategies library supports fixed sized and variable +sized collections. This can be used to optimize a wide range of +primitive data structures like arrays, lists or regular objects. Any of +these are called 'collections' in this context. The VM should have a +central class or class hierarchy for collections. In order to extend +these classes and use strategies, the library needs accessor methods for +two attributes of collection objects: strategy and storage. The easiest +way is adding the following line to the body of the root collection +class: + +:: + + rstrategies.make_accessors(strategy='strategy', storage='storage') + +This will generate the 4 accessor methods +``_[get/set]_[storage/strategy]()`` for the respective attributes. +Alternatively, implement these methods manually or overwrite the +getters/setters in ``StrategyFactory``. + +Next, the strategy classes must be defined. This requires a small class +hierarchy with a dedicated root class. In the definition of this root +class, include the following lines: + +:: + + __metaclass__ = rstrategies.StrategyMetaclass + import_from_mixin(rstrategies.AbstractStrategy) + import_from_mixin(rstrategies.SafeIndexingMixin) + +``import_from_mixin`` can be found in ``rpython.rlib.objectmodel``. If +index-checking is performed safely at other places in the VM, you can +use ``rstrategies.UnsafeIndexingMixin`` instead. If you need your own +metaclass, you can combine yours with the rstrategies one using multiple +inheritance `like +here `__. +Also implement a ``storage_factory()`` method, which returns an instance +of ``rstrategies.StorageFactory``, which is described below. + +An example ``AbstractStrategy`` class, which also stores an additional ``space`` parameter could looks like this: + +:: + + class AbstractStrategy(AbstractStrategy): + _attrs_ = ['space'] + _immutable_fields_ = ['space'] + __metaclass__ = rstrat.StrategyMetaclass + import_from_mixin(rstrat.AbstractStrategy) + import_from_mixin(rstrategies.SafeIndexingMixin) + + def __init__(self, space): + self.space = space + + def strategy_factory(self): + return self.space.strategy_factory + + +Strategy classes +---------------- + +Now you can create the actual strategy classes, subclassing them from +the single root class. The following list summarizes the basic +strategies available. + +- ``EmptyStrategy`` A strategy for empty collections; very efficient, but limited. Does not allocate anything. +- ``SingleValueStrategy`` A strategy for collections containing the same object ``n`` times. Only allocates memory to store the size of the collection. +- ``GenericStrategy`` A non-optimized strategy backed by a generic python list. This is the fallback strategy, since it can store everything, but is not optimized. +- ``WeakGenericStrategy`` Like ``GenericStrategy``, but uses ``weakref`` to hold on weakly to its elements. +- ``SingleTypeStrategy`` Can store a single unboxed type like int or float. This is the main optimizing strategy +- ``TaggingStrategy`` Extension of SingleTypeStrategy. Uses a specific value in the value range of the unboxed type to represent one additional, arbitrary object. For example, one of ``float``'s ``NaN`` representations can be used to represent special value like ``nil``. + +There are also intermediate classes, which allow creating new, more +customized strategies. For this, you should get familiar with the code. + +Include one of these mixin classes using ``import_from_mixin``. The +mixin classes contain comments describing methods or fields which are +also required in the strategy class in order to use them. Additionally, +add the ``@rstrategies.strategy(generalize=alist)`` decorator to all +strategy classes. The ``alist`` parameter must contain all strategies, +which the decorated strategy can switch to, if it can not represent a +new element anymore. +`Example `__ +for an implemented strategy. See the other strategy classes behind this +link for more examples. + +An example strategy class for optimized ``int`` storage could look like this: + +:: + + @rstrat.strategy(generalize=[GenericStrategy]) + class IntegerOrNilStrategy(AbstractStrategy): + import_from_mixin(rstrat.TaggingStrategy) + contained_type = model.W_Integer + def wrap(self, val): return self.space.wrap_int(val) + def unwrap(self, w_val): return self.space.unwrap_int(w_val) + def wrapped_tagged_value(self): return self.space.w_nil + def unwrapped_tagged_value(self): return constants.MAXINT + +Strategy Factory +---------------- + +The last part is subclassing ``rstrategies.StrategyFactory``, +overwriting the method ``instantiate_strategy`` if necessary and passing +the strategies root class to the constructor. The factory provides the +methods ``switch_strategy``, ``set_initial_strategy``, +``strategy_type_for`` which can be used by the VM code to use the +mechanism behind strategies. See the comments in the source code. + +The strategy mixins offer the following methods to manipulate the +contents of the collection: + +- basic API + + - ``size`` + +- fixed size API + + - ``store``, ``fetch``, ``slice``, ``store_all``, ``fetch_all`` + +- variable size API + + - ``insert``, ``delete``, ``append``, ``pop`` + +If the collection has a fixed size, simply never use any of the variable +size methods in the VM code. Since the strategies are singletons, these +methods need the collection object as first parameter. For convenience, +more fitting accessor methods should be implemented on the collection +class itself. + +An example strategy factory for the ``AbstractStrategy`` class above could look like this: + +:: + + class StrategyFactory(rstrategies.StrategyFactory): + _attrs_ = ['space'] + _immutable_fields_ = ['space'] + + def __init__(self, space): + self.space = space + rstrat.StrategyFactory.__init__(self, AbstractStrategy) + + def instantiate_strategy(self, strategy_type): + return strategy_type(self.space) + + def strategy_type_for(self, list_w, weak=False): + """ + Helper method for handling weak objects specially + """ + if weak: + return WeakListStrategy + return rstrategies.StrategyFactory.strategy_type_for(self, list_w) + \ No newline at end of file diff --git a/rpython/rlib/rstrategies/.coveragerc b/rpython/rlib/rstrategies/.coveragerc new file mode 100644 --- /dev/null +++ b/rpython/rlib/rstrategies/.coveragerc @@ -0,0 +1,10 @@ +# .coveragerc file to control coverage.py (code coverage plugin for pytest) +# Get it here: https://pypi.python.org/pypi/pytest-cov +# Examples: +# $ python -m pytest test --cov rpython.rlib.rstrategies --cov-report html --cov-config .coveragerc + +[run] +omit = + test/* + */__init__.py + logparser.py diff --git a/rpython/rlib/rstrategies/__init__.py b/rpython/rlib/rstrategies/__init__.py new file mode 100644 --- /dev/null +++ b/rpython/rlib/rstrategies/__init__.py @@ -0,0 +1,1 @@ +# Empy diff --git a/rpython/rlib/rstrategies/logger.py b/rpython/rlib/rstrategies/logger.py new file mode 100644 --- /dev/null +++ b/rpython/rlib/rstrategies/logger.py @@ -0,0 +1,58 @@ + +class LogEntry(object): + def __init__(self): + self.slots = 0 + self.objects = 0 + self.element_typenames = {} + + def add(self, size, element_typename): + self.slots += size + self.objects += 1 + if element_typename: + self.element_typenames[element_typename] = None + + def classnames(self): + return self.element_typenames.keys() + +class Logger(object): + _attrs_ = ["active", "aggregate", "logs"] + _immutable_fields_ = ["active?", "aggregate?", "logs"] + + def __init__(self): + self.active = False + self.aggregate = False + self.logs = {} + + def activate(self, aggregate=False): + self.active = True + self.aggregate = self.aggregate or aggregate + + def log(self, new_strategy, size, cause="", old_strategy="", typename="", element_typename=""): + if self.aggregate: + key = (cause, old_strategy, new_strategy, typename) + if key not in self.logs: + self.logs[key] = LogEntry() + entry = self.logs[key] + entry.add(size, element_typename) + else: + element_typenames = [ element_typename ] if element_typename else [] + self.output(cause, old_strategy, new_strategy, typename, size, 1, element_typenames) + + def print_aggregated_log(self): + if not self.aggregate: + return + for key, entry in self.logs.items(): + cause, old_strategy, new_strategy, typename = key + slots, objects, element_typenames = entry.slots, entry.objects, entry.classnames() + self.output(cause, old_strategy, new_strategy, typename, slots, objects, element_typenames) + + def output(self, cause, old_strategy, new_strategy, typename, slots, objects, element_typenames): + old_strategy_string = "%s -> " % old_strategy if old_strategy else "" + classname_string = " of %s" % typename if typename else "" + element_string = (" elements: " + " ".join(element_typenames)) if element_typenames else "" + format = (cause, old_strategy_string, new_strategy, classname_string, slots, objects, element_string) + self.do_print("%s (%s%s)%s size %d objects %d%s" % format) + + def do_print(self, str): + # Hook to increase testability + print str diff --git a/rpython/rlib/rstrategies/logparser.py b/rpython/rlib/rstrategies/logparser.py new file mode 100644 --- /dev/null +++ b/rpython/rlib/rstrategies/logparser.py @@ -0,0 +1,694 @@ + +import re, os, sys, operator + +""" +This script parses a log produced by rstrategies_logger.py into a graph and converts it to various outputs. +The most useful outputs are the dot* commands producing a visualization of the log using the dot-command of graphviz. +Every strategy is a node in the graph, and the edges are collections or objects that transition between +two strategies at some point during the log. +Artificial nodes are created for log entries without an explicit source node. These are the events when a +collection is created. +The input to this script is a logfile, a command and optional flags. +If the name of the logfile includes one of the AVAILABLE_VMS as a substring, the first three global variables +are automatically configured. +The script should work without these configurations, but the output will probably not be that pretty. +To avoid errors, the -a flag is implied when running without proper configuration. +""" + +# This should contain a full list of storage nodes (strategies). +# All strategies not included here will be combined into a single "Other"-node, if the -a flag is not given. +STORAGE_NODES = [] + +# This allows arbitrary renamings of storage strategy nodes +NODE_RENAMINGS = {} + +# Artificial storage-source nodes are automatically named like the associated operation. +# This dict allows customizing the names of these nodes. +STORAGE_SOURCES = {} + +def SET_VM(vm_name): + global STORAGE_NODES + global NODE_RENAMINGS + global STORAGE_SOURCES + if vm_name == 'RSqueak': + STORAGE_NODES = ['List', 'WeakList', 'SmallIntegerOrNil', 'FloatOrNil', 'AllNil'] + NODE_RENAMINGS = dict((x+'Strategy', x) for x in STORAGE_NODES) + STORAGE_SOURCES = {'Filledin': 'Image Loading', 'Initialized': 'Object Creation'} + elif vm_name == 'Pycket': + STORAGE_SOURCES = {'Created': 'Array Creation'} + # TODO + elif vm_name == 'Topaz': + # TODO + pass + else: + raise Exception("Unhandled vm name %s" % vm_name) + +AVAILABLE_VMS = ['RSqueak', 'Pycket', 'Topaz'] + +def configure_vm(logfile, flags): + vm_config_name = None + for vm_name in AVAILABLE_VMS: + if vm_name in logfile: + vm_config_name = vm_name + break + if vm_config_name is not None: + print "Using VM configuration %s" % vm_name + SET_VM(vm_name) + else: + print "No VM configuration found in filename '%s'. Available configurations: %s" % \ + (logfile, AVAILABLE_VMS) + print "Please add new VM configuration or rename logfile. Turning on -a flag to avoid errors." + flags.allstorage = True + +# ==================================================================== +# ======== Logfile parsing +# ==================================================================== + +def percent(part, total): + if total == 0: + return 0 + return float(part)*100 / total + +def parse(filename, flags, callback): + parsed_entries = 0 + if filename == "-": + opener = lambda: sys.stdin + else: + opener = lambda: open(filename, 'r', 1) + with opener() as file: + while True: + line = file.readline() + if len(line) == 0: + break + entry = parse_line(line, flags) + if entry: + parsed_entries += 1 + callback(entry) + return parsed_entries + +line_pattern = re.compile("^(?P\w+) $((?P\w+) -> )?(?P\w+)$( of (?P.+))? size (?P[0-9]+)( objects (?P[0-9]+))?( elements: (?P.+( .+)*))?$") + +def parse_line(line, flags): + result = line_pattern.match(line) + if result is None: + if flags.verbose: + print "Could not parse line: %s" % line[:-1] + return None + operation = str(result.group('operation')) + old_storage = result.group('old') + new_storage = str(result.group('new')) + classname = str(result.group('classname')) + size = int(result.group('size')) + objects = result.group('objects') + objects = int(objects) if objects else 1 + classnames = result.group('classnames') + if classnames is not None: + classnames = classnames.split(' ') + classnames = set(classnames) + else: + classnames = set() + + is_storage_source = old_storage is None + if is_storage_source: + if operation in STORAGE_SOURCES: + old_storage = STORAGE_SOURCES[operation] + else: + print "Using operation %s as storage source." % operation + old_storage = str(old_storage) + + if new_storage in NODE_RENAMINGS: + new_storage = NODE_RENAMINGS[new_storage] + if old_storage in NODE_RENAMINGS: + old_storage = NODE_RENAMINGS[old_storage] + + return LogEntry(operation, old_storage, new_storage, classname, size, objects, classnames, is_storage_source) + +class LogEntry(object): + + def __init__(self, operation, old_storage, new_storage, classname, size, objects, classnames, is_storage_source): + self.operation = operation + self.old_storage = old_storage + self.new_storage = new_storage + self.classname = classname + self.size = size + self.objects = objects + self.classnames = classnames + self.is_storage_source = is_storage_source + assert old_storage != new_storage, "old and new storage identical in log entry: %s" % self + + def full_key(self): + return (self.operation, self.old_storage, self.new_storage) + + def __lt__(self, other): + return self.classname < other.classname + + def __repr__(self): + return "%s(%s)" % (self.__str__(), object.__repr__(self)) + + def __str__(self): + old_storage_string = "%s -> " % self.old_storage if self.old_storage else "" + classname_string = " of %s" % self.classname if self.classname else "" + objects_string = " objects %d" % self.objects if self.objects > 1 else "" + return "%s (%s%s)%s size %d%s" % (self.operation, old_storage_string, self.new_storage, classname_string, self.size, objects_string) + +# ==================================================================== +# ======== Graph parsing +# ==================================================================== + +class Operations(object): + + def __init__(self, objects=0, slots=0, element_classnames=[]): + self.objects = objects + self.slots = slots + self.element_classnames = set(element_classnames) + + def __str__(self, total=None): + if self.objects == 0: + avg_slots = 0 + else: + avg_slots = float(self.slots) / self.objects + if total is not None and total.slots != 0: + percent_slots = " (%.1f%%)" % percent(self.slots, total.slots) + else: + percent_slots = "" + if total is not None and total.objects != 0: + percent_objects = " (%.1f%%)" % percent(self.objects, total.objects) + else: + percent_objects = "" + slots = format(self.slots, ",d") + objects = format(self.objects, ",d") + classnames = (" [ elements: %s ]" % ' '.join([str(x) for x in self.element_classnames])) \ + if len(self.element_classnames) else "" + return "%s%s slots in %s%s objects (avg size: %.1f)%s" % (slots, percent_slots, objects, percent_objects, avg_slots, classnames) + + def __repr__(self): + return "%s(%s)" % (self.__str__(), object.__repr__(self)) + + def add_log_entry(self, entry): + self.slots = self.slots + entry.size + self.objects = self.objects + entry.objects + self.element_classnames |= entry.classnames + + def __sub__(self, other): + return Operations(self.objects - other.objects, self.slots - other.slots) + + def __add__(self, other): + return Operations(self.objects + other.objects, self.slots + other.slots) + + def __lt__(self, other): + return self.slots < other.slots + + def empty(self): + return self.objects == 0 and self.slots == 0 + + def prefixprint(self, key="", total=None): + if not self.empty(): + print "%s%s" % (key, self.__str__(total)) + +class ClassOperations(object): + + def __init__(self): + self.classes = {} + + def cls(self, name): + if name not in self.classes: + self.classes[name] = Operations() + return self.classes[name] + + def total(self): + return reduce(operator.add, self.classes.values(), Operations()) + + def __str__(self): + return "ClassOperations(%s)" % self.classes + + def __repr__(self): + return "%s(%s)" % (self.__str__(), object.__repr__(self)) + + def __add__(self, other): + result = ClassOperations() + result.classes = dict(self.classes) + for classname, other_class in other.classes.items(): + result.cls(classname) # Make sure exists. + result.classes[classname] += other_class + return result + + def __sub__(self, other): + result = ClassOperations() + result.classes = dict(self.classes) + for classname, other_class in other.classes.items(): + result.cls(classname) # Make sure exists. + result.classes[classname] -= other_class + return result + +class StorageEdge(object): + + def __init__(self, operation="None", origin=None, target=None): + self.operation = operation + self.classes = ClassOperations() + self.origin = origin + self.target = target + self.is_storage_source = False + + def full_key(self): + return (self.operation, self.origin.name, self.target.name) + + def cls(self, classname): + return self.classes.cls(classname) + + def total(self): + return self.classes.total() + + def notify_nodes(self): + self.origin.note_outgoing(self) + self.target.note_incoming(self) + + def add_log_entry(self, entry): + self.cls(entry.classname).add_log_entry(entry) + if entry.is_storage_source: + self.is_storage_source = True + + def as_log_entries(self): + entries = [] + for classname, ops in self.classes.classes.items(): + origin = None if self.is_storage_source else self.origin.name + entry = LogEntry(self.operation, origin, self.target.name, classname, + ops.slots, ops.objects, ops.element_classnames, self.is_storage_source) + entries.append(entry) + return entries + + def __lt__(self, other): + return self.full_key() < other.full_key() + + def __str__(self): + return "[%s %s -> %s]" % (self.operation, self.origin, self.target) + + def __repr__(self): + return "%s(%s)" % (self.__str__(), object.__repr__(self)) + + def __add__(self, other): + origin = self.origin if self.origin is not None else other.origin + target = self.target if self.target is not None else other.target + result = StorageEdge(self.operation, origin, target) + result.classes += self.classes + other.classes + return result + + def __sub__(self, other): + origin = self.origin if self.origin is not None else other.origin + target = self.target if self.target is not None else other.target + result = StorageEdge(self.operation, origin, target) + result.classes += self.classes - other.classes + return result + +class StorageNode(object): + + def __init__(self, name): + self.name = name + self.incoming = set() + self.outgoing = set() + + def note_incoming(self, edge): + assert edge.target is self + if edge not in self.incoming: + self.incoming.add(edge) + + def note_outgoing(self, edge): + assert edge.origin is self + if edge not in self.outgoing: + self.outgoing.add(edge) + + def incoming_edges(self, operation): + return filter(lambda x: x.operation == operation, self.incoming) + + def outgoing_edges(self, operation): + return filter(lambda x: x.operation == operation, self.outgoing) + + def sum_incoming(self, operation): + return reduce(operator.add, self.incoming_edges(operation), StorageEdge(operation)) + + def sum_outgoing(self, operation): + return reduce(operator.add, self.outgoing_edges(operation), StorageEdge(operation)) + + def sum_all_incoming(self): + return reduce(operator.add, self.incoming, StorageEdge()) + + def sum_all_outgoing(self): + return reduce(operator.add, self.outgoing, StorageEdge()) + + def __str__(self): + return self.name + + def __repr__(self): + return "%s(%s)" % (self.__str__(), object.__repr__(self)) + + def merge_edge_sets(self, set1, set2, key_slot): + getter = lambda edge: edge.__dict__[key_slot] + set_dict = dict([(getter(edge), edge) for edge in set1]) + for edge in set2: + key = getter(edge) + if key not in set_dict: + set_dict[key] = edge + else: + set_dict[key] += edge + return set(set_dict.values()) + + def __add__(self, other): + result = StorageNode("%s %s" % (self.name, other.name)) + result.incoming = self.merge_edge_sets(self.incoming, other.incoming, "origin") + # TODO bad code + for edge in result.incoming: + edge.target = result + result.outgoing = self.merge_edge_sets(self.outgoing, other.outgoing, "target") + for edge in result.outgoing: + edge.origin = result + return result + + def __lt__(self, other): + return self.name < other.name + + def is_artificial(self): + for outgoing in self.outgoing: + if outgoing.is_storage_source: + return True + return False + + def is_storage_node(self): + return self.is_artificial() or self.name in STORAGE_NODES + + def dot_name(self): + return self.name.replace(" ", "_") + +class StorageGraph(object): + + def __init__(self): + self.nodes = {} + self.edges = {} + self.operations = set() + + def node(self, name): + if name not in self.nodes: + self.nodes[name] = StorageNode(name) + return self.nodes[name] + + def assert_sanity(self): + visited_edges = set() + for node in self.nodes.values(): + for edge in node.incoming: + assert edge in self.edges.values(), "Edge not in graph's edges: %s" % edge + visited_edges.add(edge) + if not edge.target is node: + print "Wrong edge target: %s\nIncoming edge: %s\nIn node: %s" % (edge.target, edge, node) + assert False + if not edge in edge.origin.outgoing: + print "Edge not in origin's outgoing: %s\nIncoming edge: %s\nIn node: %s" % (edge.origin.outgoing, edge, node) + assert False + for edge in node.outgoing: + assert edge in self.edges.values(), "Edge not in graph's edges: %s" % edge + visited_edges.add(edge) + if not edge.origin is node: + print "Wrong edge origin: %s\nOutgoing edge: %s\nIn node: %s" % (edge.origin, edge, node) + assert False + if not edge in edge.target.incoming: + print "Edge not in origin's incoming: %s\nOutgoing edge: %s\nIn node: %s" % (edge.target.incoming, edge, node) + assert False + assert len(visited_edges) == len(self.edges.values()), "Not all of graph's edges visited." + + def add_log_entry(self, log_entry): + self.operations.add(log_entry.operation) + key = log_entry.full_key() + if key not in self.edges: + edge = StorageEdge(log_entry.operation, self.node(log_entry.old_storage), self.node(log_entry.new_storage)) + self.edges[key] = edge + edge.notify_nodes() + self.edges[key].add_log_entry(log_entry) + + def collapse_nodes(self, collapsed_nodes, new_name=None): + if len(collapsed_nodes) == 0: + return + for node in collapsed_nodes: + del self.nodes[node.name] + for edge in node.incoming: + del self.edges[edge.full_key()] + for edge in node.outgoing: + del self.edges[edge.full_key()] + new_node = reduce(operator.add, collapsed_nodes) + if new_name is not None: + new_node.name = new_name + self.nodes[new_node.name] = new_node + # TODO bad code + for node in collapsed_nodes: + for edge in node.incoming: + edge.origin.outgoing.remove(edge) + new_edges = filter(lambda filtered: filtered.origin == edge.origin, new_node.incoming) + assert len(new_edges) == 1 + edge.origin.outgoing.add(new_edges[0]) + for edge in node.outgoing: + edge.target.incoming.remove(edge) + new_edges = filter(lambda filtered: filtered.target == edge.target, new_node.outgoing) + assert len(new_edges) == 1 + edge.target.incoming.add(new_edges[0]) + for edge in new_node.incoming: + self.edges[edge.full_key()] = edge + for edge in new_node.outgoing: + self.edges[edge.full_key()] = edge + self.assert_sanity() + + def collapse_nonstorage_nodes(self, new_name=None): + nodes = filter(lambda x: not x.is_storage_node(), self.nodes.values()) + self.collapse_nodes(nodes, new_name) + + def sorted_nodes(self): + nodes = self.nodes.values() + nodes.sort() + return nodes + +def make_graph(logfile, flags): + graph = StorageGraph() + def callback(entry): + graph.add_log_entry(entry) + parse(logfile, flags, callback) + graph.assert_sanity() + return graph + +# ==================================================================== +# ======== Command - Summarize log content +# ==================================================================== + +def command_summarize(logfile, flags): + graph = make_graph(logfile, flags) + if not flags.allstorage: + graph.collapse_nonstorage_nodes() + for node in graph.sorted_nodes(): + node.print_summary(flags, graph.operations) + +def StorageNode_print_summary(self, flags, all_operations): + print "\n%s:" % self.name + sum = StorageEdge() + total_incoming = self.sum_all_incoming().total() if flags.percent else None + + print "\tIncoming:" + for operation in all_operations: + if flags.detailed: + edges = [ (edge.origin.name, edge) for edge in self.incoming_edges(operation) ] + else: + edges = [ (operation, self.sum_incoming(operation)) ] + for edgename, edge in edges: + edge.print_with_name("\t\t\t", edgename, total_incoming, flags) + sum += edge + + print "\tOutgoing:" + for operation in all_operations: + if flags.detailed: + edges = [ (edge.target.name, edge) for edge in self.outgoing_edges(operation) ] + else: + edges = [ (operation, self.sum_outgoing(operation)) ] + for edgename, edge in edges: + edge.print_with_name("\t\t\t", edgename, total_incoming, flags) + sum -= edge + + sum.print_with_name("\t", "Remaining", total_incoming, flags) + +StorageNode.print_summary = StorageNode_print_summary + +def StorageEdge_print_with_name(self, prefix, edgename, total_reference, flags): + if flags.classes: + print "%s%s:" % (prefix, edgename) + prefix += "\t\t" + operations = self.classes.classes.items() + operations.sort(reverse=True, key=operator.itemgetter(1)) + else: + operations = [ (edgename, self.total()) ] + for classname, classops in operations: + classops.prefixprint("%s%s: " % (prefix, classname), total_reference) + +StorageEdge.print_with_name = StorageEdge_print_with_name + +# ==================================================================== +# ======== Command - DOT output +# ==================================================================== + +# Output is valid dot code and can be parsed by the graphviz dot utility. +def command_print_dot(logfile, flags): + graph = make_graph(logfile, flags) + print "/*" + print "Storage Statistics (dot format):" + print "================================" + print "*/" + print dot_string(graph, flags) + +def run_dot(logfile, flags, output_type): + import subprocess + dot = dot_string(make_graph(logfile, flags), flags) + command = ["dot", "-T%s" % output_type, "-o%s.%s" % (flags.logfile, output_type)] + print "Running:\n%s" % " ".join(command) + p = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + output = p.communicate(input=dot)[0] + print output + +def command_dot(logfile, flags): + run_dot(logfile, flags, "jpg") +def command_dot_ps(logfile, flags): + run_dot(logfile, flags, "ps") +def command_dot_pdf(logfile, flags): + run_dot(logfile, flags, "pdf") +def command_dot_svg(logfile, flags): + run_dot(logfile, flags, "svg") + +def dot_string(graph, flags): + result = "digraph G {" + incoming_cache = {} + if not flags.allstorage: + graph.collapse_nonstorage_nodes("Other") + + def make_label(edge, prefix="", total_edge=None, slots_per_object=False): + object_suffix = " objects" + slots_suffix = " slots" + if not flags.objects or not flags.slots: + object_suffix = slots_suffix = "" + if total_edge and flags.percent and total_edge.objects != 0: + percent_objects = " (%.1f%%)" % percent(edge.objects, total_edge.objects) + percent_slots = " (%.1f%%)" % percent(edge.slots, total_edge.slots) + else: + percent_objects = percent_slots = "" + label = "" + if flags.objects: + label += "%s%s%s%s
" % (prefix, format(edge.objects, ",.0f"), object_suffix, percent_objects) + if flags.slots: + label += "%s%s%s%s
" % (prefix, format(edge.slots, ",.0f"), slots_suffix, percent_slots) + if slots_per_object and flags.slotsPerObject: + label += "%.1f slots/object
" % (float(total.slots) / total.objects) + return label + + for node in graph.nodes.values(): + incoming = node.sum_all_incoming().total() + outgoing = node.sum_all_outgoing().total() + remaining = incoming - outgoing + if node.is_artificial(): + incoming_cache[node.name] = outgoing + shape = ",shape=box" + label = make_label(outgoing) + else: + incoming_cache[node.name] = incoming + shape = "" + label = make_label(incoming, "Incoming: ") + if remaining.objects != incoming.objects: + label += make_label(remaining, "Remaining: ", incoming) + result += "%s [label=<%s
%s>%s];" % (node.dot_name(), node.name, label, shape) + + for edge in graph.edges.values(): + total = edge.total() + incoming = incoming_cache[edge.origin.name] + label = make_label(total, "", incoming, slots_per_object=True) + target_node = edge.target.dot_name() + source_node = edge.origin.dot_name() + result += "%s -> %s [label=<%s>];" % (source_node, target_node, label) + + result += "}" + return result + +# ==================================================================== +# ======== Other commands +# ==================================================================== + +def command_aggregate(logfile, flags): + graph = make_graph(logfile, flags) + edges = graph.edges.values() + edges.sort() + for edge in edges: + logentries = edge.as_log_entries() + logentries.sort() + for entry in logentries: + print entry + +def command_print_entries(logfile, flags): + def callback(entry): + print entry + parse(logfile, flags, callback) + +# ==================================================================== +# ======== Main +# ==================================================================== + +class Flags(object): + + def __init__(self, flags): + self.flags = {} + for name, short in flags: + self.__dict__[name] = False + self.flags[short] = name + + def handle(self, arg): + if arg in self.flags: + self.__dict__[self.flags[arg]] = True + return True + else: + return False + + def __str__(self): + descriptions = [ ("%s (%s)" % description) for description in self.flags.items() ] + return "[%s]" % " | ".join(descriptions) + +def usage(flags, commands): + print "Arguments: logfile command %s" % flags + print "Available commands: %s" % commands + exit(1) + +def main(argv): + flags = Flags([ + # General + ('verbose', '-v'), + + # All outputs + ('percent', '-p'), + ('allstorage', '-a'), + + # Text outputs + ('detailed', '-d'), + ('classes', '-c'), + + # dot outputs + ('slots', '-s'), + ('objects', '-o'), + ('slotsPerObject', '-S'), + ]) + + command_prefix = "command_" + module = sys.modules[__name__].__dict__ + commands = [ a[len(command_prefix):] for a in module.keys() if a.startswith(command_prefix) ] + + if len(argv) < 2: + usage(flags, commands) + logfile = argv[0] + flags.logfile = logfile + configure_vm(logfile, flags) + command = argv[1] + for flag in argv[2:]: + if not flags.handle(flag): + usage(flags, commands) + if command not in commands: + usage(flags, commands) + + func = module[command_prefix + command] + func(logfile, flags) + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/rpython/rlib/rstrategies/rstrategies.py b/rpython/rlib/rstrategies/rstrategies.py new file mode 100644 --- /dev/null +++ b/rpython/rlib/rstrategies/rstrategies.py @@ -0,0 +1,572 @@ + +import weakref, sys +from rpython.rlib.rstrategies import logger +from rpython.rlib import jit, objectmodel, rerased +from rpython.rlib.objectmodel import specialize + +def make_accessors(strategy='strategy', storage='storage'): + """ + Instead of using this generator, the methods can be implemented manually. + A third way is to overwrite the getter/setter methods in StrategyFactory. + """ + def make_getter(attr): + def getter(self): return getattr(self, attr) + return getter + def make_setter(attr): + def setter(self, val): setattr(self, attr, val) + return setter + classdef = sys._getframe(1).f_locals + classdef['_get_strategy'] = make_getter(strategy) + classdef['_set_strategy'] = make_setter(strategy) + classdef['_get_storage'] = make_getter(storage) + classdef['_set_storage'] = make_setter(storage) + +class StrategyMetaclass(type): + """ + A metaclass is required, because we need certain attributes to be special + for every single strategy class. + """ + def __new__(self, name, bases, attrs): + attrs['_is_strategy'] = False + attrs['_is_singleton'] = False + attrs['_specializations'] = [] + # Not every strategy uses rerased-pairs, but they won't hurt + erase, unerase = rerased.new_erasing_pair(name) + def get_storage(self, w_self): + erased = self.strategy_factory().get_storage(w_self) + return unerase(erased) + def set_storage(self, w_self, storage): + erased = erase(storage) + self.strategy_factory().set_storage(w_self, erased) + attrs['get_storage'] = get_storage + attrs['set_storage'] = set_storage + return type.__new__(self, name, bases, attrs) + +def strategy(generalize=None, singleton=True): + """ + Strategy classes must be decorated with this. + generalize is a list of other strategies, that can be switched to from the decorated strategy. + If the singleton flag is set to False, new strategy instances will be created, + instead of always reusing the singleton object. + """ + def decorator(strategy_class): + # Patch strategy class: Add generalized_strategy_for and mark as strategy class. + if generalize: + @jit.unroll_safe + def generalized_strategy_for(self, value): + # TODO - optimize this method + for strategy in generalize: + if self.strategy_factory().strategy_singleton_instance(strategy)._check_can_handle(value): + return strategy + raise Exception("Could not find generalized strategy for %s coming from %s" % (value, self)) + strategy_class.generalized_strategy_for = generalized_strategy_for + for generalized in generalize: + generalized._specializations.append(strategy_class) + strategy_class._is_strategy = True + strategy_class._generalizations = generalize + strategy_class._is_singleton = singleton + return strategy_class + return decorator + +class StrategyFactory(object): + _immutable_fields_ = ["strategies[*]", "logger", "strategy_singleton_field"] + factory_instance_counter = 0 + + def __init__(self, root_class, all_strategy_classes=None): + if all_strategy_classes is None: + all_strategy_classes = self._collect_subclasses(root_class) + self.strategies = [] + self.logger = logger.Logger() + + # This is to avoid confusion between multiple factories existing simultaneously (e.g. in tests) + self.strategy_singleton_field = "__singleton_%i" % StrategyFactory.factory_instance_counter + StrategyFactory.factory_instance_counter += 1 + + self._create_strategy_instances(root_class, all_strategy_classes) + + def _create_strategy_instances(self, root_class, all_strategy_classes): + for strategy_class in all_strategy_classes: + if strategy_class._is_strategy: + setattr(strategy_class, self.strategy_singleton_field, self.instantiate_strategy(strategy_class)) + self.strategies.append(strategy_class) + self._patch_strategy_class(strategy_class, root_class) + self._order_strategies() + + # ============================= + # API methods + # ============================= + + def switch_strategy(self, w_self, new_strategy_type, new_element=None): + """ + Switch the strategy of w_self to the new type. + new_element can be given as as hint, purely for logging purposes. + It should be the object that was added to w_self, causing the strategy switch. + """ + old_strategy = self.get_strategy(w_self) + if new_strategy_type._is_singleton: + new_strategy = self.strategy_singleton_instance(new_strategy_type) + else: + size = old_strategy.size(w_self) + new_strategy = self.instantiate_strategy(new_strategy_type, w_self, size) + self.set_strategy(w_self, new_strategy) + old_strategy._convert_storage_to(w_self, new_strategy) + new_strategy.strategy_switched(w_self) + self.log(w_self, new_strategy, old_strategy, new_element) + return new_strategy + + def set_initial_strategy(self, w_self, strategy_type, size, elements=None): + """ + Initialize the strategy and storage fields of w_self. + This must be called before switch_strategy or any strategy method can be used. + elements is an optional list of values initially stored in w_self. + If given, then len(elements) == size must hold. + """ + assert self.get_strategy(w_self) is None, "Strategy should not be initialized yet!" + if strategy_type._is_singleton: + strategy = self.strategy_singleton_instance(strategy_type) + else: + strategy = self.instantiate_strategy(strategy_type, w_self, size) + self.set_strategy(w_self, strategy) + strategy._initialize_storage(w_self, size) + element = None + if elements: + strategy.store_all(w_self, elements) + if len(elements) > 0: element = elements[0] + strategy.strategy_switched(w_self) + self.log(w_self, strategy, None, element) + return strategy + + @jit.unroll_safe + def strategy_type_for(self, objects): + """ + Return the best-fitting strategy to hold all given objects. + """ + specialized_strategies = len(self.strategies) + can_handle = [True] * specialized_strategies + for obj in objects: + if specialized_strategies <= 1: + break + for i, strategy in enumerate(self.strategies): + if can_handle[i] and not self.strategy_singleton_instance(strategy)._check_can_handle(obj): + can_handle[i] = False + specialized_strategies -= 1 + for i, strategy_type in enumerate(self.strategies): + if can_handle[i]: + return strategy_type + raise Exception("Could not find strategy to handle: %s" % objects) + + def decorate_strategies(self, transitions): + """ + As an alternative to decorating all strategies with @strategy, + invoke this in the constructor of your StrategyFactory subclass, before + calling __init__. transitions is a dict mapping all strategy classes to + their 'generalize' list parameter (see @strategy decorator). + """ + "NOT_RPYTHON" + for strategy_class, generalized in transitions.items(): + strategy(generalized)(strategy_class) + + # ============================= + # The following methods can be overwritten to customize certain aspects of the factory. + # ============================= + + def instantiate_strategy(self, strategy_type, w_self=None, initial_size=0): + """ + Return a functional instance of strategy_type. + Overwrite this if you need a non-default constructor. + The two additional parameters should be ignored for singleton-strategies. + """ + return strategy_type() + + def log(self, w_self, new_strategy, old_strategy=None, new_element=None): + """ + This can be overwritten into a more appropriate call to self.logger.log + """ + if not self.logger.active: return + new_strategy_str = self.log_string_for_object(new_strategy) + old_strategy_str = self.log_string_for_object(old_strategy) + element_typename = self.log_string_for_object(new_element) + size = new_strategy.size(w_self) + typename = "" + cause = "Switched" if old_strategy else "Created" + self.logger.log(new_strategy_str, size, cause, old_strategy_str, typename, element_typename) + + @specialize.call_location() + def log_string_for_object(self, obj): + """ + This can be overwritten instead of the entire log() method. + Keep the specialize-annotation in order to handle different kinds of objects here. + """ + return obj.__class__.__name__ if obj else "" + + # These storage accessors are specialized because the storage field is + # populated by erased-objects which seem to be incompatible sometimes. + @specialize.call_location() + def get_storage(self, obj): + return obj._get_storage() + @specialize.call_location() + def set_storage(self, obj, val): + return obj._set_storage(val) + + def get_strategy(self, obj): + return obj._get_strategy() + def set_strategy(self, obj, val): + return obj._set_strategy(val) + + # ============================= + # Internal methods + # ============================= + + def _patch_strategy_class(self, strategy_class, root_class): + "NOT_RPYTHON" + # Patch root class: Add default handler for visitor + def _convert_storage_from_OTHER(self, w_self, previous_strategy): + self._convert_storage_from(w_self, previous_strategy) + funcname = "_convert_storage_from_" + strategy_class.__name__ + _convert_storage_from_OTHER.func_name = funcname + setattr(root_class, funcname, _convert_storage_from_OTHER) + + # Patch strategy class: Add polymorphic visitor function + def _convert_storage_to(self, w_self, new_strategy): + getattr(new_strategy, funcname)(w_self, self) + strategy_class._convert_storage_to = _convert_storage_to + + def _collect_subclasses(self, cls): + "NOT_RPYTHON" + subclasses = [] + for subcls in cls.__subclasses__(): + subclasses.append(subcls) + subclasses.extend(self._collect_subclasses(subcls)) + return subclasses + + def _order_strategies(self): + "NOT_RPYTHON" + def get_generalization_depth(strategy, visited=None): + if visited is None: + visited = set() + if strategy._generalizations: + if strategy in visited: + raise Exception("Cycle in generalization-tree of %s" % strategy) + visited.add(strategy) + depth = 0 + for generalization in strategy._generalizations: + other_depth = get_generalization_depth(generalization, set(visited)) + depth = max(depth, other_depth) + return depth + 1 + else: + return 0 + self.strategies.sort(key=get_generalization_depth, reverse=True) + + @jit.elidable + def strategy_singleton_instance(self, strategy_class): + return getattr(strategy_class, self.strategy_singleton_field) + + def _freeze_(self): + # Instance will be frozen at compile time, making accesses constant. + # The constructor does meta stuff which is not possible after translation. + return True + +class AbstractStrategy(object): + """ + == Required: + strategy_factory(self) - Access to StorageFactory + """ + + def strategy_switched(self, w_self): + # Overwrite this method for a hook whenever the strategy + # of w_self was switched to self. + pass + + # Main Fixedsize API + + def store(self, w_self, index0, value): + raise NotImplementedError("Abstract method") + + def fetch(self, w_self, index0): + raise NotImplementedError("Abstract method") + + def size(self, w_self): + raise NotImplementedError("Abstract method") + + # Fixedsize utility methods + + def slice(self, w_self, start, end): + return [ self.fetch(w_self, i) for i in range(start, end)] + + def fetch_all(self, w_self): + return self.slice(w_self, 0, self.size(w_self)) + + def store_all(self, w_self, elements): + for i, e in enumerate(elements): + self.store(w_self, i, e) + + # Main Varsize API + + def insert(self, w_self, index0, list_w): + raise NotImplementedError("Abstract method") + + def delete(self, w_self, start, end): + raise NotImplementedError("Abstract method") + + # Varsize utility methods + + def append(self, w_self, list_w): + self.insert(w_self, self.size(w_self), list_w) + + def pop(self, w_self, index0): + e = self.fetch(w_self, index0) + self.delete(w_self, index0, index0+1) + return e + + # Internal methods + + def _initialize_storage(self, w_self, initial_size): + raise NotImplementedError("Abstract method") + + def _check_can_handle(self, value): + raise NotImplementedError("Abstract method") + + def _convert_storage_to(self, w_self, new_strategy): + # This will be overwritten in _patch_strategy_class + new_strategy._convert_storage_from(w_self, self) + + @jit.unroll_safe + def _convert_storage_from(self, w_self, previous_strategy): + # This is a very unefficient (but most generic) way to do this. + # Subclasses should specialize. + storage = previous_strategy.fetch_all(w_self) + self._initialize_storage(w_self, previous_strategy.size(w_self)) + for i, field in enumerate(storage): + self.store(w_self, i, field) + + def _generalize_for_value(self, w_self, value): + strategy_type = self.generalized_strategy_for(value) + new_instance = self.strategy_factory().switch_strategy(w_self, strategy_type, new_element=value) + return new_instance + + def _cannot_handle_store(self, w_self, index0, value): + new_instance = self._generalize_for_value(w_self, value) + new_instance.store(w_self, index0, value) + + def _cannot_handle_insert(self, w_self, index0, list_w): + # TODO - optimize. Prevent multiple generalizations and slicing done by callers. + new_strategy = self._generalize_for_value(w_self, list_w[0]) + new_strategy.insert(w_self, index0, list_w) + +# ============== Special Strategies with no storage array ============== + +class EmptyStrategy(AbstractStrategy): + # == Required: + # See AbstractStrategy + + def _initialize_storage(self, w_self, initial_size): + assert initial_size == 0 + self.set_storage(w_self, None) + def _convert_storage_from(self, w_self, previous_strategy): + self.set_storage(w_self, None) + def _check_can_handle(self, value): + return False + + def fetch(self, w_self, index0): + raise IndexError + def store(self, w_self, index0, value): + self._cannot_handle_store(w_self, index0, [value]) + def insert(self, w_self, index0, list_w): + self._cannot_handle_insert(w_self, index0, list_w) + def delete(self, w_self, start, end): + self.check_index_range(w_self, start, end) + def size(self, w_self): + return 0 + +class SingleValueStrategyStorage(object): + """Small container object for a size value.""" + _attrs_ = ['size'] + def __init__(self, size=0): + self.size = size + +class SingleValueStrategy(AbstractStrategy): + # == Required: + # See AbstractStrategy + # check_index_*(...) - use mixin SafeIndexingMixin or UnsafeIndexingMixin + # value(self) - the single value contained in this strategy. Should be constant. + + def _initialize_storage(self, w_self, initial_size): + storage_obj = SingleValueStrategyStorage(initial_size) + self.set_storage(w_self, storage_obj) + def _convert_storage_from(self, w_self, previous_strategy): + self._initialize_storage(w_self, previous_strategy.size(w_self)) + def _check_can_handle(self, value): + return value is self.value() + + def fetch(self, w_self, index0): + self.check_index_fetch(w_self, index0) + return self.value() + def store(self, w_self, index0, value): + self.check_index_store(w_self, index0) + if self._check_can_handle(value): + return + self._cannot_handle_store(w_self, index0, value) + def delete(self, w_self, start, end): + self.check_index_range(w_self, start, end) + self.get_storage(w_self).size -= (end - start) + def size(self, w_self): + return self.get_storage(w_self).size + + @jit.unroll_safe + def insert(self, w_self, index0, list_w): + storage_obj = self.get_storage(w_self) + for i in range(len(list_w)): + if self._check_can_handle(list_w[i]): + storage_obj.size += 1 + else: + self._cannot_handle_insert(w_self, index0 + i, list_w[i:]) + return + +# ============== Basic strategies with storage ============== + +class StrategyWithStorage(AbstractStrategy): + # == Required: + # See AbstractStrategy + # check_index_*(...) - use mixin SafeIndexingMixin or UnsafeIndexingMixin + # default_value(self) - The value to be initially contained in this strategy + + def _initialize_storage(self, w_self, initial_size): + default = self._unwrap(self.default_value()) + self.set_storage(w_self, [default] * initial_size) + + @jit.unroll_safe + def _convert_storage_from(self, w_self, previous_strategy): + size = previous_strategy.size(w_self) + new_storage = [ self._unwrap(previous_strategy.fetch(w_self, i)) + for i in range(size) ] + self.set_storage(w_self, new_storage) + + def store(self, w_self, index0, wrapped_value): + self.check_index_store(w_self, index0) + if self._check_can_handle(wrapped_value): + unwrapped = self._unwrap(wrapped_value) + self.get_storage(w_self)[index0] = unwrapped + else: + self._cannot_handle_store(w_self, index0, wrapped_value) + + def fetch(self, w_self, index0): + self.check_index_fetch(w_self, index0) + unwrapped = self.get_storage(w_self)[index0] + return self._wrap(unwrapped) + + def _wrap(self, value): + raise NotImplementedError("Abstract method") + + def _unwrap(self, value): + raise NotImplementedError("Abstract method") + + def size(self, w_self): + return len(self.get_storage(w_self)) + + @jit.unroll_safe + def insert(self, w_self, start, list_w): + # This is following Python's behaviour - insert automatically + # happens at the beginning of an array, even if index is larger + if start > self.size(w_self): + start = self.size(w_self) + for i in range(len(list_w)): + if self._check_can_handle(list_w[i]): + self.get_storage(w_self).insert(start + i, self._unwrap(list_w[i])) + else: + self._cannot_handle_insert(w_self, start + i, list_w[i:]) + return + + def delete(self, w_self, start, end): + self.check_index_range(w_self, start, end) + assert start >= 0 and end >= 0 + del self.get_storage(w_self)[start : end] + +class GenericStrategy(StrategyWithStorage): + # == Required: + # See StrategyWithStorage + + def _wrap(self, value): + return value + def _unwrap(self, value): + return value + def _check_can_handle(self, wrapped_value): + return True + +class WeakGenericStrategy(StrategyWithStorage): + # == Required: + # See StrategyWithStorage + + def _wrap(self, value): + return value() or self.default_value() + def _unwrap(self, value): + assert value is not None + return weakref.ref(value) + def _check_can_handle(self, wrapped_value): + return True + +# ============== Mixins for index checking operations ============== + +class SafeIndexingMixin(object): + def check_index_store(self, w_self, index0): + self.check_index(w_self, index0) + def check_index_fetch(self, w_self, index0): + self.check_index(w_self, index0) + def check_index_range(self, w_self, start, end): + if end < start: + raise IndexError + self.check_index(w_self, start) + self.check_index(w_self, end) + def check_index(self, w_self, index0): + if index0 < 0 or index0 >= self.size(w_self): + raise IndexError + +class UnsafeIndexingMixin(object): + def check_index_store(self, w_self, index0): + pass + def check_index_fetch(self, w_self, index0): + pass + def check_index_range(self, w_self, start, end): + pass + +# ============== Specialized Storage Strategies ============== + +class SpecializedStrategy(StrategyWithStorage): + # == Required: + # See StrategyWithStorage + # wrap(self, value) - Return a boxed object for the primitive value + # unwrap(self, value) - Return the unboxed primitive value of value + + def _unwrap(self, value): + return self.unwrap(value) + def _wrap(self, value): + return self.wrap(value) + +class SingleTypeStrategy(SpecializedStrategy): + # == Required Functions: + # See SpecializedStrategy + # contained_type - The wrapped type that can be stored in this strategy + + def _check_can_handle(self, value): + return isinstance(value, self.contained_type) + +class TaggingStrategy(SingleTypeStrategy): + """This strategy uses a special tag value to represent a single additional object.""" + # == Required: + # See SingleTypeStrategy + # wrapped_tagged_value(self) - The tagged object + # unwrapped_tagged_value(self) - The unwrapped tag value representing the tagged object + + def _check_can_handle(self, value): + return value is self.wrapped_tagged_value() or \ + (isinstance(value, self.contained_type) and \ + self.unwrap(value) != self.unwrapped_tagged_value()) + + def _unwrap(self, value): + if value is self.wrapped_tagged_value(): + return self.unwrapped_tagged_value() + return self.unwrap(value) + + def _wrap(self, value): + if value == self.unwrapped_tagged_value(): + return self.wrapped_tagged_value() + return self.wrap(value) diff --git a/rpython/rlib/rstrategies/test/test_rstrategies.py b/rpython/rlib/rstrategies/test/test_rstrategies.py new file mode 100644 --- /dev/null +++ b/rpython/rlib/rstrategies/test/test_rstrategies.py @@ -0,0 +1,552 @@ + +import py +from rpython.rlib.rstrategies import rstrategies as rs +from rpython.rlib.objectmodel import import_from_mixin + +# === Define small model tree + +class W_AbstractObject(object): + pass + +class W_Object(W_AbstractObject): + pass + +class W_Integer(W_AbstractObject): + def __init__(self, value): + self.value = value + def __eq__(self, other): + return isinstance(other, W_Integer) and self.value == other.value + +class W_List(W_AbstractObject): + rs.make_accessors() + def __init__(self, strategy=None, size=0, elements=None): + self.strategy = None + if strategy: + factory.set_initial_strategy(self, strategy, size, elements) + def fetch(self, i): + assert self.strategy + return self.strategy.fetch(self, i) + def store(self, i, value): + assert self.strategy + return self.strategy.store(self, i, value) + def size(self): + assert self.strategy + return self.strategy.size(self) + def insert(self, index0, list_w): + assert self.strategy + return self.strategy.insert(self, index0, list_w) + def delete(self, start, end): + assert self.strategy + return self.strategy.delete(self, start, end) + def append(self, list_w): + assert self.strategy + return self.strategy.append(self, list_w) + def pop(self, index0): + assert self.strategy + return self.strategy.pop(self, index0) + def slice(self, start, end): + assert self.strategy + return self.strategy.slice(self, start, end) + def fetch_all(self): + assert self.strategy + return self.strategy.fetch_all(self) + def store_all(self, elements): + assert self.strategy + return self.strategy.store_all(self, elements) + +w_nil = W_Object() + +# === Define concrete strategy classes + +class AbstractStrategy(object): + __metaclass__ = rs.StrategyMetaclass + import_from_mixin(rs.AbstractStrategy) + import_from_mixin(rs.SafeIndexingMixin) + def __init__(self, factory, w_self=None, size=0): + self.factory = factory + def strategy_factory(self): + return self.factory + +class Factory(rs.StrategyFactory): + switching_log = [] + + def __init__(self, root_class): + self.decorate_strategies({ + EmptyStrategy: [NilStrategy, IntegerStrategy, IntegerOrNilStrategy, GenericStrategy], + NilStrategy: [IntegerOrNilStrategy, GenericStrategy], + GenericStrategy: [], + IntegerStrategy: [IntegerOrNilStrategy, GenericStrategy], + IntegerOrNilStrategy: [GenericStrategy], + }) + rs.StrategyFactory.__init__(self, root_class) + + def instantiate_strategy(self, strategy_type, w_self=None, size=0): + return strategy_type(self, w_self, size) + + def set_strategy(self, w_list, strategy): + old_strategy = self.get_strategy(w_list) + self.switching_log.append((old_strategy, strategy)) + super(Factory, self).set_strategy(w_list, strategy) + + def clear_log(self): + del self.switching_log[:] + +class EmptyStrategy(AbstractStrategy): + import_from_mixin(rs.EmptyStrategy) + # TODO - implement and test transition from Generic back to Empty + +class NilStrategy(AbstractStrategy): + import_from_mixin(rs.SingleValueStrategy) + def value(self): return w_nil + +class GenericStrategy(AbstractStrategy): + import_from_mixin(rs.GenericStrategy) + import_from_mixin(rs.UnsafeIndexingMixin) + def default_value(self): return w_nil + +class WeakGenericStrategy(AbstractStrategy): + import_from_mixin(rs.WeakGenericStrategy) + def default_value(self): return w_nil + +class IntegerStrategy(AbstractStrategy): + import_from_mixin(rs.SingleTypeStrategy) + contained_type = W_Integer + def wrap(self, value): return W_Integer(value) + def unwrap(self, value): return value.value + def default_value(self): return W_Integer(0) + +class IntegerOrNilStrategy(AbstractStrategy): + import_from_mixin(rs.TaggingStrategy) + contained_type = W_Integer + def wrap(self, value): return W_Integer(value) + def unwrap(self, value): return value.value + def default_value(self): return w_nil + def wrapped_tagged_value(self): return w_nil + def unwrapped_tagged_value(self): import sys; return sys.maxint + + at rs.strategy(generalize=[], singleton=False) +class NonSingletonStrategy(GenericStrategy): + def __init__(self, factory, w_list=None, size=0): + super(NonSingletonStrategy, self).__init__(factory, w_list, size) + self.w_list = w_list + self.the_size = size + +class NonStrategy(NonSingletonStrategy): + pass + + at rs.strategy(generalize=[]) +class InefficientStrategy(GenericStrategy): + def _convert_storage_from(self, w_self, previous_strategy): + return AbstractStrategy._convert_storage_from(self, w_self, previous_strategy) + +factory = Factory(AbstractStrategy) + +def check_contents(list, expected): + assert list.size() == len(expected) + for i, val in enumerate(expected): + assert list.fetch(i) == val + +def teardown(): + factory.clear_log() + +# === Test Initialization and fetch + +def test_setup(): + pass + +def test_factory_setup(): + expected_strategies = 7 + assert len(factory.strategies) == expected_strategies + assert len(set(factory.strategies)) == len(factory.strategies) + for strategy in factory.strategies: + assert isinstance(factory.strategy_singleton_instance(strategy), strategy) + +def test_factory_setup_singleton_instances(): + new_factory = Factory(AbstractStrategy) + s1 = factory.strategy_singleton_instance(GenericStrategy) + s2 = new_factory.strategy_singleton_instance(GenericStrategy) + assert s1 is not s2 + assert s1.strategy_factory() is factory + assert s2.strategy_factory() is new_factory + +def test_metaclass(): + assert NonStrategy._is_strategy == False + assert IntegerOrNilStrategy._is_strategy == True + assert IntegerOrNilStrategy._is_singleton == True + assert NonSingletonStrategy._is_singleton == False + assert NonStrategy._is_singleton == False + assert NonStrategy.get_storage is not NonSingletonStrategy.get_storage + +def test_singletons(): + def do_test_singletons(cls, expected_true): + l1 = W_List(cls, 0) + l2 = W_List(cls, 0) + if expected_true: + assert l1.strategy is l2.strategy + else: + assert l1.strategy is not l2.strategy + do_test_singletons(EmptyStrategy, True) + do_test_singletons(NonSingletonStrategy, False) + do_test_singletons(NonStrategy, False) + do_test_singletons(GenericStrategy, True) + +def do_test_initialization(cls, default_value=w_nil, is_safe=True): + size = 10 + l = W_List(cls, size) + s = l.strategy + assert s.size(l) == size + assert s.fetch(l,0) == default_value + assert s.fetch(l,size/2) == default_value + assert s.fetch(l,size-1) == default_value + py.test.raises(IndexError, s.fetch, l, size) + py.test.raises(IndexError, s.fetch, l, size+1) + py.test.raises(IndexError, s.fetch, l, size+5) + if is_safe: + py.test.raises(IndexError, s.fetch, l, -1) + else: + assert s.fetch(l, -1) == s.fetch(l, size - 1) + +def test_init_Empty(): + l = W_List(EmptyStrategy, 0) + s = l.strategy + assert s.size(l) == 0 + py.test.raises(IndexError, s.fetch, l, 0) + py.test.raises(IndexError, s.fetch, l, 10) + py.test.raises(IndexError, s.delete, l, 0, 1) + py.test.raises(AssertionError, W_List, EmptyStrategy, 2) # Only size 0 possible. + +def test_init_Nil(): + do_test_initialization(NilStrategy) + +def test_init_Generic(): + do_test_initialization(GenericStrategy, is_safe=False) + +def test_init_WeakGeneric(): + do_test_initialization(WeakGenericStrategy) + +def test_init_Integer(): + do_test_initialization(IntegerStrategy, default_value=W_Integer(0)) + +def test_init_IntegerOrNil(): + do_test_initialization(IntegerOrNilStrategy) + +# === Test Simple store + +def do_test_store(cls, stored_value=W_Object(), is_safe=True, is_varsize=False): + size = 10 + l = W_List(cls, size) + s = l.strategy + def store_test(index): + s.store(l, index, stored_value) + assert s.fetch(l, index) == stored_value + store_test(0) + store_test(size/2) + store_test(size-1) + if not is_varsize: + py.test.raises(IndexError, s.store, l, size, stored_value) + py.test.raises(IndexError, s.store, l, size+1, stored_value) + py.test.raises(IndexError, s.store, l, size+5, stored_value) + if is_safe: + py.test.raises(IndexError, s.store, l, -1, stored_value) + else: + store_test(-1) + +def test_store_Nil(): + do_test_store(NilStrategy, stored_value=w_nil) + +def test_store_Generic(): + do_test_store(GenericStrategy, is_safe=False) + +def test_store_WeakGeneric(): + do_test_store(WeakGenericStrategy, stored_value=w_nil) + +def test_store_Integer(): + do_test_store(IntegerStrategy, stored_value=W_Integer(100)) + +def test_store_IntegerOrNil(): + do_test_store(IntegerOrNilStrategy, stored_value=W_Integer(100)) + do_test_store(IntegerOrNilStrategy, stored_value=w_nil) + +# === Test Insert + +def do_test_insert(cls, values): + l = W_List(cls, 0) + assert len(values) >= 6 + values0 = values[0:1] + values1 = values[1:2] + values2 = values[2:4] + values3 = values[4:6] + l.insert(3, values0) # Will still be inserted at the very beginning + check_contents(l, values0) + l.insert(1, values1+values3) + check_contents(l, values0+values1+values3) + l.insert(2, values2) + check_contents(l, values) + return l + +def test_insert_Nil(): + do_test_insert(NilStrategy, [w_nil]*6) + +def test_insert_Generic(): + do_test_insert(GenericStrategy, [W_Object() for _ in range(6)]) + +def test_insert_WeakGeneric(): + do_test_insert(WeakGenericStrategy, [W_Object() for _ in range(6)]) + +def test_insert_Integer(): + do_test_insert(IntegerStrategy, [W_Integer(x) for x in range(6)]) + +def test_insert_IntegerOrNil(): + do_test_insert(IntegerOrNilStrategy, [w_nil]+[W_Integer(x) for x in range(4)]+[w_nil]) + do_test_insert(IntegerOrNilStrategy, [w_nil]*6) + +# === Test Delete + +def do_test_delete(cls, values, indexing_unsafe=False): + assert len(values) >= 6 + l = W_List(cls, len(values), values) + if not indexing_unsafe: + py.test.raises(IndexError, l.delete, 2, 1) + l.delete(2, 4) + del values[2: 4] + check_contents(l, values) + l.delete(1, 2) + del values[1: 2] + check_contents(l, values) + +def test_delete_Nil(): + do_test_delete(NilStrategy, [w_nil]*6) + +def test_delete_Generic(): + do_test_delete(GenericStrategy, [W_Object() for _ in range(6)], indexing_unsafe=True) + +def test_delete_WeakGeneric(): + do_test_delete(WeakGenericStrategy, [W_Object() for _ in range(6)]) + +def test_delete_Integer(): + do_test_delete(IntegerStrategy, [W_Integer(x) for x in range(6)]) + +def test_delete_IntegerOrNil(): + do_test_delete(IntegerOrNilStrategy, [w_nil]+[W_Integer(x) for x in range(4)]+[w_nil]) + do_test_delete(IntegerOrNilStrategy, [w_nil]*6) + +# === Test Transitions + +def test_CheckCanHandle(): + def assert_handles(cls, good, bad): + s = cls(0) + for val in good: + assert s._check_can_handle(val) + for val in bad: + assert not s._check_can_handle(val) + obj = W_Object() + i = W_Integer(0) + nil = w_nil + + assert_handles(EmptyStrategy, [], [nil, obj, i]) + assert_handles(NilStrategy, [nil], [obj, i]) + assert_handles(GenericStrategy, [nil, obj, i], []) + assert_handles(WeakGenericStrategy, [nil, obj, i], []) + assert_handles(IntegerStrategy, [i], [nil, obj]) + assert_handles(IntegerOrNilStrategy, [nil, i], [obj]) + +def do_test_transition(OldStrategy, value, NewStrategy, initial_size=10): + w = W_List(OldStrategy, initial_size) + old = w.strategy + w.store(0, value) + assert isinstance(w.strategy, NewStrategy) + assert factory.switching_log == [(None, old), (old, w.strategy)] + +def test_AllNil_to_Generic(): + do_test_transition(NilStrategy, W_Object(), GenericStrategy) + +def test_AllNil_to_IntegerOrNil(): + do_test_transition(NilStrategy, W_Integer(0), IntegerOrNilStrategy) + +def test_IntegerOrNil_to_Generic(): + do_test_transition(IntegerOrNilStrategy, W_Object(), GenericStrategy) + +def test_Integer_to_IntegerOrNil(): + do_test_transition(IntegerStrategy, w_nil, IntegerOrNilStrategy) + +def test_Generic_to_AllNil(): + w = W_List(GenericStrategy, 5) + old = w.strategy + factory.switch_strategy(w, NilStrategy) + assert isinstance(w.strategy, NilStrategy) + assert factory.switching_log == [(None, old), (old, w.strategy)] + +def test_Integer_Generic(): + do_test_transition(IntegerStrategy, W_Object(), GenericStrategy) + +def test_TaggingValue_not_storable(): + tag = IntegerOrNilStrategy(10).unwrapped_tagged_value() # sys.maxint + do_test_transition(IntegerOrNilStrategy, W_Integer(tag), GenericStrategy) + +def test_insert_StrategySwitch_IntOrNil(): + o = W_Object() + l = do_test_insert(IntegerOrNilStrategy, [W_Integer(1), w_nil, o, o, w_nil, W_Integer(3)]) + assert isinstance(l.strategy, GenericStrategy) + +def test_insert_StrategySwitch_AllNil(): + o = W_Object() + l = do_test_insert(NilStrategy, [w_nil, w_nil, o, o, w_nil, w_nil]) + assert isinstance(l.strategy, GenericStrategy) + +def test_transition_to_nonSingleton(): + l = W_List(NilStrategy, 5) + factory.switch_strategy(l, NonSingletonStrategy) + strategy1 = l.strategy + assert isinstance(strategy1, NonSingletonStrategy) + factory.switch_strategy(l, NonSingletonStrategy) + assert strategy1 != l.strategy From noreply at buildbot.pypy.org Wed Apr 22 15:09:53 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 22 Apr 2015 15:09:53 +0200 (CEST) Subject: [pypy-commit] pypy vmprof2: fix test_compile Message-ID: <20150422130953.015A61C02A3@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof2 Changeset: r76884:a8d35ae5a9a4 Date: 2015-04-21 20:05 +0200 http://bitbucket.org/pypy/pypy/changeset/a8d35ae5a9a4/ Log: fix test_compile diff --git a/rpython/jit/metainterp/test/test_compile.py b/rpython/jit/metainterp/test/test_compile.py --- a/rpython/jit/metainterp/test/test_compile.py +++ b/rpython/jit/metainterp/test/test_compile.py @@ -19,7 +19,8 @@ ts = typesystem.llhelper def __init__(self): self.seen = [] - def compile_loop(self, inputargs, operations, token, log=True, name='', + def compile_loop(self, inputargs, operations, token, jd_id=0, + unique_id=0, log=True, name='', logger=None): token.compiled_loop_token = self.Storage() self.seen.append((inputargs, operations, token)) @@ -38,6 +39,9 @@ def attach_unoptimized_bridge_from_interp(*args): pass + def get_unique_id(*args): + return 0 + def get_location_str(self, args): return 'location' @@ -59,6 +63,7 @@ class FakeMetaInterp: call_pure_results = {} class jitdriver_sd: + index = 0 warmstate = FakeState() virtualizable_info = None From noreply at buildbot.pypy.org Wed Apr 22 15:09:54 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 22 Apr 2015 15:09:54 +0200 (CEST) Subject: [pypy-commit] pypy vmprof2: fix test_pypyjit Message-ID: <20150422130954.317B51C02A3@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof2 Changeset: r76885:9629a3afd70c Date: 2015-04-22 15:09 +0200 http://bitbucket.org/pypy/pypy/changeset/9629a3afd70c/ Log: fix test_pypyjit diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py --- a/pypy/module/pypyjit/interp_resop.py +++ b/pypy/module/pypyjit/interp_resop.py @@ -105,7 +105,7 @@ ofs = ops_offset.get(op, 0) if op.opnum == rop.DEBUG_MERGE_POINT: jd_sd = jitdrivers_sd[op.getarg(0).getint()] - greenkey = op.getarglist()[4:] + greenkey = op.getarglist()[3:] repr = jd_sd.warmstate.get_location_str(greenkey) w_greenkey = wrap_greenkey(space, jd_sd.jitdriver, greenkey, repr) l_w.append(DebugMergePoint(space, jit_hooks._cast_to_gcref(op), diff --git a/pypy/module/pypyjit/test/test_jit_hook.py b/pypy/module/pypyjit/test/test_jit_hook.py --- a/pypy/module/pypyjit/test/test_jit_hook.py +++ b/pypy/module/pypyjit/test/test_jit_hook.py @@ -55,7 +55,7 @@ oplist = parse(""" [i1, i2, p2] i3 = int_add(i1, i2) - debug_merge_point(0, 0, 0, 0, 0, 0, ConstPtr(ptr0)) + debug_merge_point(0, 0, 0, 0, 0, ConstPtr(ptr0)) guard_nonnull(p2) [] guard_true(i3) [] """, namespace={'ptr0': code_gcref}).operations From noreply at buildbot.pypy.org Wed Apr 22 15:10:24 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 22 Apr 2015 15:10:24 +0200 (CEST) Subject: [pypy-commit] pypy default: A test and fix for rewrite.py. Message-ID: <20150422131024.956901C02A3@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76886:70a22aaa3a7c Date: 2015-04-22 15:10 +0200 http://bitbucket.org/pypy/pypy/changeset/70a22aaa3a7c/ Log: A test and fix for rewrite.py. diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -62,6 +62,10 @@ op = operations[i] if op.getopnum() == rop.DEBUG_MERGE_POINT: continue + # ---------- GETFIELD_GC ---------- + if op.getopnum() == rop.GETFIELD_GC: + self.handle_getfield_gc(op) + continue # ---------- turn NEWxxx into CALL_MALLOC_xxx ---------- if op.is_malloc(): self.handle_malloc_operation(op) @@ -122,6 +126,18 @@ # ---------- + def handle_getfield_gc(self, op): + """See test_zero_ptr_field_before_getfield(). We hope there is + no getfield_gc in the middle of initialization code, but there + shouldn't be, given that a 'new' is already delayed by previous + optimization steps. In practice it should immediately be + followed by a bunch of 'setfields', and the 'pending_zeros' + optimization we do here is meant for this case.""" + self.emit_pending_zeros() + self.newops.append(op) + + # ---------- + def handle_malloc_operation(self, op): opnum = op.getopnum() if opnum == rop.NEW: diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py --- a/rpython/jit/backend/llsupport/test/test_rewrite.py +++ b/rpython/jit/backend/llsupport/test/test_rewrite.py @@ -1031,3 +1031,21 @@ guard_false(i1, descr=guarddescr) [] jump() """) + + def test_zero_ptr_field_before_getfield(self): + # This case may need to be fixed in the metainterp/optimizeopt + # already so that it no longer occurs for rewrite.py. But anyway + # it's a good idea to make sure rewrite.py is correct on its own. + self.check_rewrite(""" + [] + p0 = new(descr=tdescr) + p1 = getfield_gc(p0, descr=tdescr) + jump(p1) + """, """ + [] + p0 = call_malloc_nursery(%(tdescr.size)d) + setfield_gc(p0, 5678, descr=tiddescr) + zero_ptr_field(p0, %(tdescr.gc_fielddescrs[0].offset)s) + p1 = getfield_gc(p0, descr=tdescr) + jump(p1) + """) From noreply at buildbot.pypy.org Wed Apr 22 15:26:08 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 22 Apr 2015 15:26:08 +0200 (CEST) Subject: [pypy-commit] pypy vmprof2: fix this test Message-ID: <20150422132608.F2AC31C069E@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof2 Changeset: r76887:647fe5a61ba2 Date: 2015-04-22 15:25 +0200 http://bitbucket.org/pypy/pypy/changeset/647fe5a61ba2/ Log: fix this test diff --git a/pypy/module/_vmprof/test/test_direct.py b/pypy/module/_vmprof/test/test_direct.py --- a/pypy/module/_vmprof/test/test_direct.py +++ b/pypy/module/_vmprof/test/test_direct.py @@ -29,7 +29,7 @@ { long c = *current_pos_addr; if (c >= 5) - return 0; + return -1; *current_pos_addr = c + 1; return *((long*)codemap_raw + c); } @@ -57,10 +57,11 @@ lib.buffer[2] = 12 lib.buffer[3] = 16 lib.buffer[4] = 0 - buf = ffi.new("long[5]", [0] * 5) + buf = ffi.new("long[10]", [0] * 10) result = ffi.cast("void**", buf) res = lib.vmprof_write_header_for_jit_addr(result, 0, ffi.NULL, 100) - assert res == 3 - assert buf[0] == 16 - assert buf[1] == 12 - assert buf[2] == 8 + assert res == 6 + assert buf[0] == 2 + assert buf[1] == 16 + assert buf[2] == 12 + assert buf[3] == 8 From noreply at buildbot.pypy.org Wed Apr 22 15:41:06 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 22 Apr 2015 15:41:06 +0200 (CEST) Subject: [pypy-commit] pypy vmprof2: Move _code_unique_id out of ExecutionContext into its own small Message-ID: <20150422134106.E40571C140B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof2 Changeset: r76888:cc4843b3d8b2 Date: 2015-04-22 15:41 +0200 http://bitbucket.org/pypy/pypy/changeset/cc4843b3d8b2/ Log: Move _code_unique_id out of ExecutionContext into its own small singleton: it is a global counter, not a per-thread one. diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -11,7 +11,7 @@ INT_MIN, INT_MAX, UINT_MAX, USHRT_MAX from pypy.interpreter.executioncontext import (ExecutionContext, ActionFlag, - UserDelAction) + UserDelAction, CodeUniqueIds) from pypy.interpreter.error import OperationError, new_exception_class, oefmt from pypy.interpreter.argument import Arguments from pypy.interpreter.miscutils import ThreadLocals, make_weak_value_dictionary @@ -388,6 +388,7 @@ self.actionflag = ActionFlag() # changed by the signal module self.check_signal_action = None # changed by the signal module self.user_del_action = UserDelAction(self) + self.code_unique_ids = CodeUniqueIds() self._code_of_sys_exc_info = None # can be overridden to a subclass @@ -667,15 +668,15 @@ return ec def register_code_callback(self, callback): - ec = self.getexecutioncontext() - ec._code_callback = callback + cui = self.code_unique_ids + cui.code_callback = callback def register_code_object(self, pycode): - ec = self.getexecutioncontext() - if ec._code_callback is None: + cui = self.code_unique_ids + if cui.code_callback is None: return - ec._code_callback(self, pycode) - + cui.code_callback(self, pycode) + def _freeze_(self): return True diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -2,7 +2,6 @@ from pypy.interpreter.error import OperationError, get_cleared_operation_error from rpython.rlib.unroll import unrolling_iterable from rpython.rlib import jit -from rpython.rlib.objectmodel import we_are_translated TICK_COUNTER_STEP = 100 @@ -35,16 +34,6 @@ self.w_profilefuncarg = None self.thread_disappeared = False # might be set to True after os.fork() - if sys.maxint == 2147483647: - self._code_unique_id = 0 # XXX this is wrong, it won't work on 32bit - else: - if we_are_translated(): - self._code_unique_id = 0x7000000000000000 - else: - self._code_unique_id = 0x7700000000000000 - # should be enough code objects - self._code_callback = None - @staticmethod def _mark_thread_disappeared(space): # Called in the child process after os.fork() by interp_posix.py. @@ -590,3 +579,11 @@ # there is no list of length n: if n is large, then the GC # will run several times while walking the list, but it will # see lower and lower memory usage, with no lower bound of n. + +class CodeUniqueIds(object): + def __init__(self): + if sys.maxint == 2147483647: + self.code_unique_id = 0 # XXX this is wrong, it won't work on 32bit + else: + self.code_unique_id = 0x7000000000000000 + self.code_callback = None diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -127,9 +127,9 @@ from pypy.objspace.std.mapdict import init_mapdict_cache init_mapdict_cache(self) - ec = self.space.getexecutioncontext() - self._unique_id = ec._code_unique_id - ec._code_unique_id += 4 # so we have two bits that we can mark stuff + cui = self.space.code_unique_ids + self._unique_id = cui.code_unique_id + cui.code_unique_id += 4 # so we have two bits that we can mark stuff # with def _get_full_name(self): From noreply at buildbot.pypy.org Wed Apr 22 15:51:43 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 22 Apr 2015 15:51:43 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: in-progress: declare structs on FFIs after a cdef() Message-ID: <20150422135143.11D201C1468@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1775:8acbc3c22b22 Date: 2015-04-22 15:52 +0200 http://bitbucket.org/cffi/cffi/changeset/8acbc3c22b22/ Log: in-progress: declare structs on FFIs after a cdef() diff --git a/cffi/api.py b/cffi/api.py --- a/cffi/api.py +++ b/cffi/api.py @@ -97,6 +97,7 @@ if override: for cache in self._function_caches: cache.clear() + _set_cdef_types(self) def dlopen(self, name, flags=0): """Load and return a dynamic library identified by 'name'. @@ -565,3 +566,27 @@ else: with ffi._lock: return ffi._get_cached_btype(tp) + +def _set_cdef_types(ffi): + struct_unions = [] + pending_completion = [] + for name, tp in sorted(ffi._parser._declarations.items()): + if name.startswith('struct '): + tp.check_not_partial() + basename = name[7:] + BType = _cffi1_backend.new_struct_type(basename) + struct_unions.append(basename) + struct_unions.append(BType) + if tp.fldtypes is not None: + pending_completion.append((tp, BType)) + # + ffi.__set_types(struct_unions) + # + for tp, BType in pending_completion: + fldtypes = [ffi.typeof(ftp._get_c_name()) for ftp in tp.fldtypes] + lst = list(zip(tp.fldnames, fldtypes, tp.fldbitsize)) + sflags = 0 + if tp.packed: + sflags = 8 # SF_PACKED + _cffi1_backend.complete_struct_or_union(BType, lst, ffi, + -1, -1, sflags) diff --git a/new/cffi1_module.c b/new/cffi1_module.c --- a/new/cffi1_module.c +++ b/new/cffi1_module.c @@ -51,7 +51,7 @@ if (m == NULL) return -1; - FFIObject *ffi = ffi_internal_new(&FFI_Type, ctx, 1); + FFIObject *ffi = ffi_internal_new(&FFI_Type, ctx); Py_XINCREF(ffi); /* make the ffi object really immortal */ if (ffi == NULL || PyModule_AddObject(m, "ffi", (PyObject *)ffi) < 0) return -1; diff --git a/new/ffi_obj.c b/new/ffi_obj.c --- a/new/ffi_obj.c +++ b/new/ffi_obj.c @@ -24,15 +24,15 @@ struct _cffi_parse_info_s info; int ctx_is_static; builder_c_t *types_builder; + PyObject *dynamic_types; _cffi_opcode_t internal_output[FFI_COMPLEXITY_OUTPUT]; }; static FFIObject *ffi_internal_new(PyTypeObject *ffitype, - const struct _cffi_type_context_s *ctx, - int ctx_is_static) + const struct _cffi_type_context_s *static_ctx) { FFIObject *ffi; - if (ctx_is_static) { + if (static_ctx != NULL) { ffi = (FFIObject *)PyObject_GC_New(FFIObject, ffitype); /* we don't call PyObject_GC_Track() here: from _cffi_init_module() it is not needed, because in this case the ffi object is immortal */ @@ -43,16 +43,17 @@ if (ffi == NULL) return NULL; - ffi->types_builder = new_builder_c(ctx); + ffi->types_builder = new_builder_c(static_ctx); if (ffi->types_builder == NULL) { Py_DECREF(ffi); return NULL; } ffi->gc_wrefs = NULL; - ffi->info.ctx = ctx; + ffi->info.ctx = &ffi->types_builder->ctx; ffi->info.output = ffi->internal_output; ffi->info.output_size = FFI_COMPLEXITY_OUTPUT; - ffi->ctx_is_static = ctx_is_static; + ffi->ctx_is_static = (static_ctx != NULL); + ffi->dynamic_types = NULL; return ffi; } @@ -60,6 +61,7 @@ { PyObject_GC_UnTrack(ffi); Py_XDECREF(ffi->gc_wrefs); + Py_XDECREF(ffi->dynamic_types); if (!ffi->ctx_is_static) free_builder_c(ffi->types_builder); @@ -77,22 +79,7 @@ static PyObject *ffiobj_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { /* user-facing initialization code, for explicit FFI() calls */ - struct _cffi_type_context_s *ctx; - PyObject *result; - - ctx = PyMem_Malloc(sizeof(struct _cffi_type_context_s)); - if (ctx == NULL) { - PyErr_NoMemory(); - return NULL; - } - memset(ctx, 0, sizeof(struct _cffi_type_context_s)); - - result = (PyObject *)ffi_internal_new(type, ctx, 0); - if (result == NULL) { - PyMem_Free(ctx); - return NULL; - } - return result; + return (PyObject *)ffi_internal_new(type, NULL); } static int ffiobj_init(PyObject *self, PyObject *args, PyObject *kwds) @@ -548,7 +535,72 @@ return 0; } +static PyObject *ffi__set_types(FFIObject *self, PyObject *args) +{ + PyObject *lst1; + _cffi_opcode_t *types = NULL; + struct _cffi_struct_union_s *struct_unions = NULL; + + if (!PyArg_ParseTuple(args, "O!", &PyList_Type, &lst1)) + return NULL; + + if (self->ctx_is_static) { + bad_usage: + PyMem_Free(struct_unions); + PyMem_Free(types); + if (!PyErr_Occurred()) + PyErr_SetString(PyExc_RuntimeError, "internal error"); + return NULL; + } + + cleanup_builder_c(self->types_builder); + + int i, lst_length = PyList_GET_SIZE(lst1) / 2; + Py_ssize_t new_size_1 = sizeof(_cffi_opcode_t) * lst_length; + Py_ssize_t new_size_2 = sizeof(struct _cffi_struct_union_s) * lst_length; + types = PyMem_Malloc(new_size_1); + struct_unions = PyMem_Malloc(new_size_2); + if (!types || !struct_unions) { + PyErr_NoMemory(); + goto bad_usage; + } + memset(types, 0, new_size_1); + memset(struct_unions, 0, new_size_2); + + for (i = 0; i < lst_length; i++) { + PyObject *x = PyList_GET_ITEM(lst1, i * 2); + if (!PyString_Check(x)) + goto bad_usage; + struct_unions[i].name = PyString_AS_STRING(x); + struct_unions[i].type_index = i; + //struct_unions[i].flags = ...; + struct_unions[i].size = (size_t)-2; + struct_unions[i].alignment = -2; + + x = PyList_GET_ITEM(lst1, i * 2 + 1); + if (!CTypeDescr_Check(x)) + goto bad_usage; + types[i] = x; + } + for (i = 0; i < lst_length; i++) { + PyObject *x = (PyObject *)types[i]; + Py_INCREF(x); + } + + Py_INCREF(lst1); /* to keep alive the strings in '.name' */ + Py_XDECREF(self->dynamic_types); + self->dynamic_types = lst1; + self->types_builder->ctx.types = types; + self->types_builder->num_types_imported = lst_length; + self->types_builder->ctx.struct_unions = struct_unions; + self->types_builder->ctx.num_struct_unions = lst_length; + + Py_INCREF(Py_None); + return Py_None; +} + static PyMethodDef ffi_methods[] = { + {"__set_types", (PyCFunction)ffi__set_types,METH_VARARGS}, #if 0 {"addressof", (PyCFunction)ffi_addressof, METH_VARARGS}, #endif diff --git a/new/realize_c_type.c b/new/realize_c_type.c --- a/new/realize_c_type.c +++ b/new/realize_c_type.c @@ -2,6 +2,7 @@ typedef struct { struct _cffi_type_context_s ctx; /* inlined substructure */ PyObject *types_dict; + int num_types_imported; } builder_c_t; @@ -44,9 +45,15 @@ return err; } -static void free_builder_c(builder_c_t *builder) +static void cleanup_builder_c(builder_c_t *builder) { - Py_XDECREF(builder->types_dict); + int i; + for (i = builder->num_types_imported; (--i) >= 0; ) { + _cffi_opcode_t x = builder->ctx.types[i]; + if ((((uintptr_t)x) & 1) == 0) { + Py_XDECREF((PyObject *)x); + } + } const void *mem[] = {builder->ctx.types, builder->ctx.globals, @@ -54,11 +61,16 @@ builder->ctx.fields, builder->ctx.enums, builder->ctx.typenames}; - int i; for (i = 0; i < sizeof(mem) / sizeof(*mem); i++) { if (mem[i] != NULL) PyMem_Free((void *)mem[i]); } +} + +static void free_builder_c(builder_c_t *builder) +{ + Py_XDECREF(builder->types_dict); + cleanup_builder_c(builder); PyMem_Free(builder); } @@ -74,8 +86,13 @@ PyErr_NoMemory(); return NULL; } - builder->ctx = *ctx; + if (ctx) + builder->ctx = *ctx; + else + memset(&builder->ctx, 0, sizeof(builder->ctx)); + builder->types_dict = ldict; + builder->num_types_imported = 0; return builder; } diff --git a/new/test_dlopen.py b/new/test_dlopen.py --- a/new/test_dlopen.py +++ b/new/test_dlopen.py @@ -3,6 +3,11 @@ import math +def test_cdef_struct(): + ffi = FFI() + ffi.cdef("struct foo_s { int a, b; };") + assert ffi.sizeof("struct foo_s") == 8 + def test_math_sin(): py.test.skip("XXX redo!") ffi = FFI() From noreply at buildbot.pypy.org Wed Apr 22 16:10:02 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 22 Apr 2015 16:10:02 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: fix Message-ID: <20150422141002.2B36D1C02A3@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1776:9972564d7f2c Date: 2015-04-22 16:10 +0200 http://bitbucket.org/cffi/cffi/changeset/9972564d7f2c/ Log: fix diff --git a/cffi/api.py b/cffi/api.py --- a/cffi/api.py +++ b/cffi/api.py @@ -572,12 +572,11 @@ pending_completion = [] for name, tp in sorted(ffi._parser._declarations.items()): if name.startswith('struct '): - tp.check_not_partial() basename = name[7:] BType = _cffi1_backend.new_struct_type(basename) struct_unions.append(basename) struct_unions.append(BType) - if tp.fldtypes is not None: + if not tp.partial and tp.fldtypes is not None: pending_completion.append((tp, BType)) # ffi.__set_types(struct_unions) From noreply at buildbot.pypy.org Wed Apr 22 16:13:12 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 22 Apr 2015 16:13:12 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: cdef() unions Message-ID: <20150422141312.7C17A1C0661@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1777:f19eb734b709 Date: 2015-04-22 16:13 +0200 http://bitbucket.org/cffi/cffi/changeset/f19eb734b709/ Log: cdef() unions diff --git a/cffi/api.py b/cffi/api.py --- a/cffi/api.py +++ b/cffi/api.py @@ -571,9 +571,12 @@ struct_unions = [] pending_completion = [] for name, tp in sorted(ffi._parser._declarations.items()): - if name.startswith('struct '): - basename = name[7:] - BType = _cffi1_backend.new_struct_type(basename) + kind, basename = name.split(' ', 1) + if kind == 'struct' or kind == 'union': + if kind == 'struct': + BType = _cffi1_backend.new_struct_type(basename) + else: + BType = _cffi1_backend.new_union_type(basename) struct_unions.append(basename) struct_unions.append(BType) if not tp.partial and tp.fldtypes is not None: diff --git a/new/ffi_obj.c b/new/ffi_obj.c --- a/new/ffi_obj.c +++ b/new/ffi_obj.c @@ -573,14 +573,14 @@ goto bad_usage; struct_unions[i].name = PyString_AS_STRING(x); struct_unions[i].type_index = i; - //struct_unions[i].flags = ...; - struct_unions[i].size = (size_t)-2; - struct_unions[i].alignment = -2; x = PyList_GET_ITEM(lst1, i * 2 + 1); if (!CTypeDescr_Check(x)) goto bad_usage; types[i] = x; + struct_unions[i].flags = ((CTypeDescrObject *)x)->ct_flags & CT_UNION; + struct_unions[i].size = (size_t)-2; + struct_unions[i].alignment = -2; } for (i = 0; i < lst_length; i++) { PyObject *x = (PyObject *)types[i]; diff --git a/new/test_dlopen.py b/new/test_dlopen.py --- a/new/test_dlopen.py +++ b/new/test_dlopen.py @@ -8,6 +8,11 @@ ffi.cdef("struct foo_s { int a, b; };") assert ffi.sizeof("struct foo_s") == 8 +def test_cdef_union(): + ffi = FFI() + ffi.cdef("union foo_s { int a, b; };") + assert ffi.sizeof("union foo_s") == 4 + def test_math_sin(): py.test.skip("XXX redo!") ffi = FFI() From noreply at buildbot.pypy.org Wed Apr 22 16:14:31 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 22 Apr 2015 16:14:31 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: pass one more line of the next test Message-ID: <20150422141431.39D8A1C0661@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1778:e3d0c9fc52cb Date: 2015-04-22 16:15 +0200 http://bitbucket.org/cffi/cffi/changeset/e3d0c9fc52cb/ Log: pass one more line of the next test diff --git a/new/test_verify1.py b/new/test_verify1.py --- a/new/test_verify1.py +++ b/new/test_verify1.py @@ -1,5 +1,5 @@ import sys, math, py -from cffi1 import FFI, VerificationError, VerificationMissing, model +from cffi1 import FFI, VerificationError, model lib_m = ['m'] if sys.platform == 'win32': @@ -450,9 +450,9 @@ ...; }; """) - py.test.raises(VerificationMissing, ffi.sizeof, 'struct foo_s') - py.test.raises(VerificationMissing, ffi.offsetof, 'struct foo_s', 'x') - py.test.raises(VerificationMissing, ffi.new, 'struct foo_s *') + py.test.raises(ffi.error, ffi.sizeof, 'struct foo_s') + py.test.raises(ffi.error, ffi.offsetof, 'struct foo_s', 'x') + py.test.raises(ffi.error, ffi.new, 'struct foo_s *') ffi.verify(""" struct foo_s { int a, b, x, c, d, e; From noreply at buildbot.pypy.org Wed Apr 22 16:27:11 2015 From: noreply at buildbot.pypy.org (mjacob) Date: Wed, 22 Apr 2015 16:27:11 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: Change some whitespace to make a new revision to test something on the new build slave. Message-ID: <20150422142711.C3AE51C06CD@cobra.cs.uni-duesseldorf.de> Author: Manuel Jacob Branch: py3.3 Changeset: r76889:d27cff3ec7ad Date: 2015-04-22 16:26 +0200 http://bitbucket.org/pypy/pypy/changeset/d27cff3ec7ad/ Log: Change some whitespace to make a new revision to test something on the new build slave. diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -13,7 +13,7 @@ class CConfig: _compilation_info_ = ExternalCompilationInfo( - includes=['sys/stat.h', + includes=['sys/stat.h', 'unistd.h', 'fcntl.h'], ) From noreply at buildbot.pypy.org Wed Apr 22 16:39:33 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 22 Apr 2015 16:39:33 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: ffi.offsetof() Message-ID: <20150422143933.471721C02A3@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1779:3f61ed84336c Date: 2015-04-22 16:40 +0200 http://bitbucket.org/cffi/cffi/changeset/3f61ed84336c/ Log: ffi.offsetof() diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -4952,27 +4952,26 @@ return res; } -static PyObject *b_typeoffsetof(PyObject *self, PyObject *args) -{ - PyObject *res, *fieldname; - CTypeDescrObject *ct; +static CTypeDescrObject *direct_typeoffsetof(CTypeDescrObject *ct, + PyObject *fieldname, + int following, Py_ssize_t *offset) +{ + /* Does not return a new reference! */ + CTypeDescrObject *res; CFieldObject *cf; - Py_ssize_t offset; - int following = 0; - - if (!PyArg_ParseTuple(args, "O!O|i:typeoffsetof", - &CTypeDescr_Type, &ct, &fieldname, &following)) - return NULL; if (PyTextAny_Check(fieldname)) { if (!following && (ct->ct_flags & CT_POINTER)) ct = ct->ct_itemdescr; - if (!(ct->ct_flags & (CT_STRUCT|CT_UNION)) || - force_lazy_struct(ct) <= 0) { + if (!(ct->ct_flags & (CT_STRUCT|CT_UNION))) { + PyErr_SetString(PyExc_TypeError, + "with a field name argument, expected a " + "struct or union ctype"); + return NULL; + } + if (force_lazy_struct(ct) <= 0) { if (!PyErr_Occurred()) - PyErr_SetString(PyExc_TypeError, - "with a field name argument, expected an " - "initialized struct or union ctype"); + PyErr_SetString(PyExc_TypeError, "struct/union is opaque"); return NULL; } cf = (CFieldObject *)PyDict_GetItem(ct->ct_stuff, fieldname); @@ -4984,8 +4983,8 @@ PyErr_SetString(PyExc_TypeError, "not supported for bitfields"); return NULL; } - res = (PyObject *)cf->cf_type; - offset = cf->cf_offset; + res = cf->cf_type; + *offset = cf->cf_offset; } else { ssize_t index = PyInt_AsSsize_t(fieldname); @@ -5002,14 +5001,32 @@ "pointer to non-opaque"); return NULL; } - res = (PyObject *)ct->ct_itemdescr; - offset = index * ct->ct_itemdescr->ct_size; - if ((offset / ct->ct_itemdescr->ct_size) != index) { + res = ct->ct_itemdescr; + *offset = index * ct->ct_itemdescr->ct_size; + if ((*offset / ct->ct_itemdescr->ct_size) != index) { PyErr_SetString(PyExc_OverflowError, "array offset would overflow a Py_ssize_t"); return NULL; } } + return res; +} + +static PyObject *b_typeoffsetof(PyObject *self, PyObject *args) +{ + PyObject *res, *fieldname; + CTypeDescrObject *ct; + Py_ssize_t offset; + int following = 0; + + if (!PyArg_ParseTuple(args, "O!O|i:typeoffsetof", + &CTypeDescr_Type, &ct, &fieldname, &following)) + return NULL; + + res = (PyObject *)direct_typeoffsetof(ct, fieldname, following, &offset); + if (res == NULL) + return NULL; + return Py_BuildValue("(On)", res, offset); } diff --git a/new/ffi_obj.c b/new/ffi_obj.c --- a/new/ffi_obj.c +++ b/new/ffi_obj.c @@ -290,58 +290,46 @@ "string or unicode string.\n" "\n" "If 'cdata' is an enum, returns the value of the enumerator as a\n" -"string, or 'NUMBER' if the value is out of range.\n"); +"string, or 'NUMBER' if the value is out of range."); #define ffi_string b_string /* ffi_string() => b_string() from _cffi_backend.c */ -#if 0 -static CFieldObject *_ffi_field(CTypeDescrObject *ct, const char *fieldname) +PyDoc_STRVAR(ffi_offsetof_doc, +"Return the offset of the named field inside the given structure or\n" +"array, which must be given as a C type name. You can give several\n" +"field names in case of nested structures. You can also give numeric\n" +"values which correspond to array items, in case of an array type."); + +static PyObject *ffi_offsetof(FFIObject *self, PyObject *args) { - CFieldObject *cf; - if (force_lazy_struct(ct) == NULL) { - PyErr_Format(PyExc_TypeError, "'%s' is incomplete", ct->ct_name); + PyObject *arg; + CTypeDescrObject *ct; + Py_ssize_t i, offset; + + if (PyTuple_Size(args) < 2) { + PyErr_SetString(PyExc_TypeError, + "offsetof() expects at least 2 arguments"); return NULL; } - cf = (CFieldObject *)PyDict_GetItemString(ct->ct_stuff, fieldname); - if (cf == NULL) { - PyErr_Format(PyExc_KeyError, "'%s' has got no field '%s'", - ct->ct_name, fieldname); - return NULL; - } - if (cf->cf_bitshift >= 0) { - PyErr_SetString(PyExc_TypeError, "not supported for bitfields"); - return NULL; - } - return cf; -} -static PyObject *ffi_offsetof(ZefFFIObject *self, PyObject *args) -{ - PyObject *arg; - char *fieldname; - CTypeDescrObject *ct; - CFieldObject *cf; - - if (!PyArg_ParseTuple(args, "Os:offsetof", &arg, &fieldname)) - return NULL; - + arg = PyTuple_GET_ITEM(args, 0); ct = _ffi_type(self, arg, ACCEPT_STRING|ACCEPT_CTYPE); if (ct == NULL) return NULL; - if (!(ct->ct_flags & (CT_STRUCT|CT_UNION))) { - PyErr_Format(PyExc_TypeError, - "expected a struct or union ctype, got '%s'", - ct->ct_name); - return NULL; + offset = 0; + for (i = 1; i < PyTuple_GET_SIZE(args); i++) { + Py_ssize_t ofs1; + ct = direct_typeoffsetof(ct, PyTuple_GET_ITEM(args, i), i > 1, &ofs1); + if (ct == NULL) + return NULL; + offset += ofs1; } - cf = _ffi_field(ct, fieldname); - if (cf == NULL) - return NULL; - return PyInt_FromSsize_t(cf->cf_offset); + return PyInt_FromSsize_t(offset); } +#if 0 static PyObject *ffi_addressof(ZefFFIObject *self, PyObject *args) { PyObject *obj; @@ -612,8 +600,8 @@ {"gc", (PyCFunction)ffi_gc, METH_VARARGS}, {"getctype", (PyCFunction)ffi_getctype, METH_VARARGS}, {"load_library", (PyCFunction)ffi_load_library,METH_VARARGS|METH_KEYWORDS}, - {"offsetof", (PyCFunction)ffi_offsetof, METH_VARARGS}, #endif + {"offsetof", (PyCFunction)ffi_offsetof, METH_VARARGS,ffi_offsetof_doc}, {"new", (PyCFunction)ffi_new, METH_VARARGS, ffi_new_doc}, #if 0 {"new_handle", (PyCFunction)ffi_new_handle,METH_O}, diff --git a/new/test_recompiler.py b/new/test_recompiler.py --- a/new/test_recompiler.py +++ b/new/test_recompiler.py @@ -216,6 +216,9 @@ py.test.raises(OverflowError, "p.b -= 1") q = ffi.new("struct bar_s *", {'f': p}) assert q.f == p + # + assert ffi.offsetof("struct foo_s", "a") == 0 + assert ffi.offsetof("struct foo_s", "b") == 4 def test_verify_exact_field_offset(): ffi = FFI() diff --git a/new/test_verify1.py b/new/test_verify1.py --- a/new/test_verify1.py +++ b/new/test_verify1.py @@ -451,8 +451,8 @@ }; """) py.test.raises(ffi.error, ffi.sizeof, 'struct foo_s') - py.test.raises(ffi.error, ffi.offsetof, 'struct foo_s', 'x') - py.test.raises(ffi.error, ffi.new, 'struct foo_s *') + py.test.raises(TypeError, ffi.offsetof, 'struct foo_s', 'x') + py.test.raises(TypeError, ffi.new, 'struct foo_s *') ffi.verify(""" struct foo_s { int a, b, x, c, d, e; From noreply at buildbot.pypy.org Wed Apr 22 17:38:27 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 22 Apr 2015 17:38:27 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: in-progress: bitfield support Message-ID: <20150422153827.E97501C0D78@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1780:ded8861f09fd Date: 2015-04-22 17:39 +0200 http://bitbucket.org/cffi/cffi/changeset/ded8861f09fd/ Log: in-progress: bitfield support diff --git a/new/realize_c_type.c b/new/realize_c_type.c --- a/new/realize_c_type.c +++ b/new/realize_c_type.c @@ -466,11 +466,16 @@ int i; for (i = 0; i < s->num_fields; i++, fld++) { _cffi_opcode_t op = fld->field_type_op; + int fbitsize = -1; PyObject *f; CTypeDescrObject *ctf; switch (_CFFI_GETOP(op)) { + case _CFFI_OP_BITFIELD: + assert(fld->field_size >= 0); + fbitsize = (int)fld->field_size; + /* fall-through */ case _CFFI_OP_NOOP: ctf = realize_c_type(builder, builder->ctx.types, _CFFI_GETARG(op)); @@ -484,8 +489,9 @@ if (fld->field_offset == (size_t)-1) { /* unnamed struct, with field positions and sizes entirely - determined by complete_struct_or_union() and not checked */ - assert(fld->field_size == -1); + determined by complete_struct_or_union() and not checked. + Or, bitfields (field_size >= 0), similarly not checked. */ + assert(fld->field_size == (size_t)-1 || fbitsize >= 0); } else if (detect_custom_layout(ct, SF_STD_FIELD_POS, ctf->ct_size, fld->field_size, @@ -494,7 +500,7 @@ return -1; f = Py_BuildValue("(sOin)", fld->name, ctf, - (int)-1, (Py_ssize_t)fld->field_offset); + fbitsize, (Py_ssize_t)fld->field_offset); if (f == NULL) { Py_DECREF(fields); return -1; diff --git a/new/recompiler.py b/new/recompiler.py --- a/new/recompiler.py +++ b/new/recompiler.py @@ -439,25 +439,29 @@ flags = ('|'.join(flags)) or '0' if tp.fldtypes is not None: c_field = [approxname] - for fldname, fldtype in zip(tp.fldnames, tp.fldtypes): + for fldname, fldtype, fbitsize in tp.enumfields(): fldtype = self._field_type(tp, fldname, fldtype) spaces = " " * len(fldname) # cname is None for _add_missing_struct_unions() only - if cname is None or ( + op = '_CFFI_OP_NOOP' + if fbitsize >= 0: + op = '_CFFI_OP_BITFIELD' + size = '%d /* bits */' % fbitsize + elif cname is None or ( isinstance(fldtype, model.ArrayType) and fldtype.length is None): size = '(size_t)-1' else: size = 'sizeof(((%s)0)->%s)' % (tp.get_c_name('*'), fldname) - if cname is None: + if cname is None or fbitsize >= 0: offset = '(size_t)-1' else: offset = 'offsetof(%s, %s)' % (tp.get_c_name(''), fldname) c_field.append( ' { "%s", %s,\n' % (fldname, offset) + ' %s %s,\n' % (spaces, size) + - ' %s _CFFI_OP(_CFFI_OP_NOOP, %s) },' % ( - spaces, self._typesdict[fldtype])) + ' %s _CFFI_OP(%s, %s) },' % ( + spaces, op, self._typesdict[fldtype])) self._lsts["field"].append('\n'.join(c_field)) # if cname is None: # unknown name, for _add_missing_struct_unions From noreply at buildbot.pypy.org Wed Apr 22 17:46:01 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 22 Apr 2015 17:46:01 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Fix Message-ID: <20150422154601.2E4771C02A3@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1781:538f36a9670b Date: 2015-04-22 17:46 +0200 http://bitbucket.org/cffi/cffi/changeset/538f36a9670b/ Log: Fix diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -4084,8 +4084,9 @@ as 1 instead. But for ctypes support, we allow the manually- specified totalsize to be zero in this case. */ boffsetmax = (boffsetmax + 7) / 8; /* bits -> bytes */ + boffsetmax = (boffsetmax + alignment - 1) & ~(alignment-1); if (totalsize < 0) { - totalsize = (boffsetmax + alignment - 1) & ~(alignment-1); + totalsize = boffsetmax; if (totalsize == 0) totalsize = 1; } From noreply at buildbot.pypy.org Wed Apr 22 18:21:15 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 22 Apr 2015 18:21:15 +0200 (CEST) Subject: [pypy-commit] pypy vmprof2: try to fix those tests Message-ID: <20150422162115.345BA1C1334@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof2 Changeset: r76890:233ac16e55d6 Date: 2015-04-22 18:21 +0200 http://bitbucket.org/pypy/pypy/changeset/233ac16e55d6/ Log: try to fix those tests diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py --- a/pypy/module/pypyjit/test_pypy_c/model.py +++ b/pypy/module/pypyjit/test_pypy_c/model.py @@ -134,7 +134,8 @@ def _ops_for_chunk(self, chunk, include_guard_not_invalidated): for op in chunk.operations: - if op.name != 'debug_merge_point' and \ + if op.name not in ('debug_merge_point', 'enter_portal_frame', + 'leave_portal_frame') and \ (op.name != 'guard_not_invalidated' or include_guard_not_invalidated): yield op From noreply at buildbot.pypy.org Wed Apr 22 21:53:14 2015 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 22 Apr 2015 21:53:14 +0200 (CEST) Subject: [pypy-commit] pypy object-dtype2: test, fix for numpy object scalar actually becoming the contained object Message-ID: <20150422195314.8B0591C1334@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: object-dtype2 Changeset: r76891:e91a14676362 Date: 2015-04-22 22:07 +0300 http://bitbucket.org/pypy/pypy/changeset/e91a14676362/ Log: test, fix for numpy object scalar actually becoming the contained object diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -202,12 +202,16 @@ return self elif isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool() \ and w_idx.ndims() > 0: - return self.getitem_filter(space, w_idx) + w_ret = self.getitem_filter(space, w_idx) else: try: - return self.implementation.descr_getitem(space, self, w_idx) + w_ret = self.implementation.descr_getitem(space, self, w_idx) except ArrayArgumentException: - return self.getitem_array_int(space, w_idx) + w_ret = self.getitem_array_int(space, w_idx) + if isinstance(w_ret, boxes.W_ObjectBox): + #return the W_Root object, not a scalar + w_ret = w_ret.w_obj + return w_ret def getitem(self, space, index_list): return self.implementation.getitem_index(space, index_list) diff --git a/pypy/module/micronumpy/test/test_object_arrays.py b/pypy/module/micronumpy/test/test_object_arrays.py --- a/pypy/module/micronumpy/test/test_object_arrays.py +++ b/pypy/module/micronumpy/test/test_object_arrays.py @@ -117,14 +117,15 @@ b = np.object_(3) b2 = np.object_(3.0) c = np.object_([4, 5]) - d = np.object_([None, {}, []]) - print type(a) + d = np.array([None])[0] assert a is None assert type(b) is int assert type(b2) is float assert type(c) is np.ndarray assert c.dtype == object - assert d.dtype == object + assert type(d) is type(None) + e = np.object_([None, {}, []]) + assert e.dtype == object def test_mem_array_creation_invalid_specification(self): # while not specifically testing object dtype, this From noreply at buildbot.pypy.org Wed Apr 22 21:53:15 2015 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 22 Apr 2015 21:53:15 +0200 (CEST) Subject: [pypy-commit] pypy object-dtype2: skip test_zjit pending moving the tests to test_micronumpy in test_pypy_c Message-ID: <20150422195315.B55D01C1334@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: object-dtype2 Changeset: r76892:ec7e1a7a0847 Date: 2015-04-22 22:53 +0300 http://bitbucket.org/pypy/pypy/changeset/ec7e1a7a0847/ Log: skip test_zjit pending moving the tests to test_micronumpy in test_pypy_c diff --git a/pypy/goal/targetnumpystandalone.py b/pypy/goal/targetnumpystandalone.py deleted file mode 100644 --- a/pypy/goal/targetnumpystandalone.py +++ /dev/null @@ -1,43 +0,0 @@ - -""" Usage: - -./targetnumpystandalone-c array_size - -Will execute a give numpy bytecode. Arrays will be ranges (in float) modulo 10, -constants would be consecutive starting from one. - -Bytecode should contain letters 'a' 'l' and 'f' so far and be correct -""" - -import time -from pypy.module.micronumpy.compile import numpy_compile -from rpython.jit.codewriter.policy import JitPolicy -from rpython.rtyper.annlowlevel import hlstr - -def entry_point(argv): - if len(argv) != 3: - print __doc__ - return 1 - try: - size = int(argv[2]) - except ValueError: - print "INVALID LITERAL FOR INT:", argv[2] - print __doc__ - return 3 - t0 = time.time() - main(argv[0], size) - print "bytecode:", argv[0], "size:", size - print "took:", time.time() - t0 - return 0 - -def main(bc, size): - if not isinstance(bc, str): - bc = hlstr(bc) # for tests - a = numpy_compile(bc, size) - a = a.compute() - -def target(*args): - return entry_point, None - -def jitpolicy(driver): - return JitPolicy() diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -9,6 +9,7 @@ from pypy.module.micronumpy.compile import FakeSpace, Parser, InterpreterState from pypy.module.micronumpy.base import W_NDimArray +py.test.skip('move these to pypyjit/test_pypy_c/test_micronumpy') class TestNumpyJit(LLJitMixin): graph = None From noreply at buildbot.pypy.org Wed Apr 22 23:38:28 2015 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 22 Apr 2015 23:38:28 +0200 (CEST) Subject: [pypy-commit] pypy vmprof2: try a fix for msvc on win32 Message-ID: <20150422213828.6919E1C0F93@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: vmprof2 Changeset: r76893:9591d90505b9 Date: 2015-04-23 00:38 +0300 http://bitbucket.org/pypy/pypy/changeset/9591d90505b9/ Log: try a fix for msvc on win32 diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py --- a/rpython/jit/backend/llsupport/codemap.py +++ b/rpython/jit/backend/llsupport/codemap.py @@ -26,7 +26,16 @@ srcdir = os.path.join(os.path.dirname(__file__), 'src') eci = ExternalCompilationInfo(post_include_bits=[""" +#if defined _MSC_VER && _MSC_VER < 1600 +#ifdef _WIN32 +typedef unsigned int uintptr_t; +#else +typedef usigned long uintptr_t; +#endif +#else #include +#endif + RPY_EXTERN long pypy_jit_codemap_add(uintptr_t addr, unsigned int machine_code_size, long *bytecode_info, diff --git a/rpython/jit/backend/llsupport/src/skiplist.c b/rpython/jit/backend/llsupport/src/skiplist.c --- a/rpython/jit/backend/llsupport/src/skiplist.c +++ b/rpython/jit/backend/llsupport/src/skiplist.c @@ -1,5 +1,25 @@ #include +#if defined _MSC_VER + #if _MSC_VER < 1600 + #include + int __sync_lock_test_and_set(int * i, int j) + { + return _interlockedbittestandreset(i, j); + } + int __sync_lock_release(int *i) + { + return _interlockedbittestandreset(i, 0); + } + #ifdef _WIN32 + typedef unsigned int uintptr_t; + #else + typedef usigned long uintptr_t; + #endif + #endif +#else #include +#endif + #define HAS_SKIPLIST #define SKIPLIST_HEIGHT 8 From noreply at buildbot.pypy.org Wed Apr 22 23:44:54 2015 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 22 Apr 2015 23:44:54 +0200 (CEST) Subject: [pypy-commit] pypy vmprof2: fix bad logic Message-ID: <20150422214454.634821C1370@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: vmprof2 Changeset: r76894:df219e810ccb Date: 2015-04-23 00:45 +0300 http://bitbucket.org/pypy/pypy/changeset/df219e810ccb/ Log: fix bad logic diff --git a/rpython/jit/backend/llsupport/src/skiplist.c b/rpython/jit/backend/llsupport/src/skiplist.c --- a/rpython/jit/backend/llsupport/src/skiplist.c +++ b/rpython/jit/backend/llsupport/src/skiplist.c @@ -1,15 +1,15 @@ #include #if defined _MSC_VER + #include + int __sync_lock_test_and_set(int * i, int j) + { + return _interlockedbittestandreset(i, j); + } + int __sync_lock_release(int *i) + { + return _interlockedbittestandreset(i, 0); + } #if _MSC_VER < 1600 - #include - int __sync_lock_test_and_set(int * i, int j) - { - return _interlockedbittestandreset(i, j); - } - int __sync_lock_release(int *i) - { - return _interlockedbittestandreset(i, 0); - } #ifdef _WIN32 typedef unsigned int uintptr_t; #else From noreply at buildbot.pypy.org Thu Apr 23 00:26:45 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 23 Apr 2015 00:26:45 +0200 (CEST) Subject: [pypy-commit] pypy default: encode unicode dtype spec to ascii Message-ID: <20150422222645.458DD1C0F93@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76895:2301f0bf4a68 Date: 2015-04-23 01:24 +0300 http://bitbucket.org/pypy/pypy/changeset/2301f0bf4a68/ Log: encode unicode dtype spec to ascii diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py --- a/pypy/module/micronumpy/descriptor.py +++ b/pypy/module/micronumpy/descriptor.py @@ -545,6 +545,8 @@ space.gettypefor(boxes.W_VoidBox), shape=shape, subdtype=subdtype, elsize=size) + if space.isinstance_w(w_dtype, space.w_unicode): + w_dtype = space.wrap(space.str_w(w_dtype)) # may raise if invalid if space.is_none(w_dtype): return cache.w_float64dtype elif space.isinstance_w(w_dtype, w_subtype): diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -58,6 +58,7 @@ assert exc.value[0] == "there are no fields defined" assert dtype('int8').num == 1 + assert dtype(u'int8').num == 1 assert dtype('int8').name == 'int8' assert dtype('void').name == 'void' assert dtype(int).fields is None From noreply at buildbot.pypy.org Thu Apr 23 04:25:05 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Thu, 23 Apr 2015 04:25:05 +0200 (CEST) Subject: [pypy-commit] pypy default: fix 2301f0bf4a68 Message-ID: <20150423022505.E15531C140B@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76896:a007e985b031 Date: 2015-04-23 03:24 +0100 http://bitbucket.org/pypy/pypy/changeset/a007e985b031/ Log: fix 2301f0bf4a68 diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py --- a/pypy/module/micronumpy/descriptor.py +++ b/pypy/module/micronumpy/descriptor.py @@ -545,13 +545,13 @@ space.gettypefor(boxes.W_VoidBox), shape=shape, subdtype=subdtype, elsize=size) - if space.isinstance_w(w_dtype, space.w_unicode): - w_dtype = space.wrap(space.str_w(w_dtype)) # may raise if invalid if space.is_none(w_dtype): return cache.w_float64dtype - elif space.isinstance_w(w_dtype, w_subtype): + if space.isinstance_w(w_dtype, w_subtype): return w_dtype - elif space.isinstance_w(w_dtype, space.w_str): + if space.isinstance_w(w_dtype, space.w_unicode): + w_dtype = space.wrap(space.str_w(w_dtype)) # may raise if invalid + if space.isinstance_w(w_dtype, space.w_str): name = space.str_w(w_dtype) if _check_for_commastring(name): return dtype_from_spec(space, w_dtype) From noreply at buildbot.pypy.org Thu Apr 23 05:56:43 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Thu, 23 Apr 2015 05:56:43 +0200 (CEST) Subject: [pypy-commit] pypy can_cast: create stub for np.can_cast() and copy test from numpy Message-ID: <20150423035643.769351C0661@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: can_cast Changeset: r76897:32155f8c346e Date: 2015-04-23 02:03 +0100 http://bitbucket.org/pypy/pypy/changeset/32155f8c346e/ Log: create stub for np.can_cast() and copy test from numpy diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py --- a/pypy/module/micronumpy/__init__.py +++ b/pypy/module/micronumpy/__init__.py @@ -21,6 +21,7 @@ 'count_nonzero': 'arrayops.count_nonzero', 'dot': 'arrayops.dot', 'result_type': 'arrayops.result_type', + 'can_cast': 'arrayops.can_cast', 'where': 'arrayops.where', 'set_string_function': 'appbridge.set_string_function', diff --git a/pypy/module/micronumpy/arrayops.py b/pypy/module/micronumpy/arrayops.py --- a/pypy/module/micronumpy/arrayops.py +++ b/pypy/module/micronumpy/arrayops.py @@ -310,3 +310,7 @@ space.call_function(space.gettypefor(descriptor.W_Dtype), w_arg)) result = ufuncs.find_binop_result_dtype(space, result, dtype) return result + + at unwrap_spec(casting=str) +def can_cast(space, w_from, w_totype, casting='safe'): + return space.w_True diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -1817,7 +1817,7 @@ s[...] = 2 v = s.view(x.__class__) assert (v == 2).all() - + def test_tolist_scalar(self): from numpy import dtype int32 = dtype('int32').type @@ -3973,3 +3973,80 @@ a = ndarray._from_shape_and_storage((2,), addr, int, sz, strides=[2 * base.strides[0]]) assert a[1] == 3 + + def test_can_cast(self): + import numpy as np + + assert np.can_cast(np.int32, np.int64) + assert np.can_cast(np.float64, complex) + assert not np.can_cast(np.complex64, float) + + assert np.can_cast('i8', 'f8') + assert not np.can_cast('i8', 'f4') + assert np.can_cast('i4', 'S11') + + assert np.can_cast('i8', 'i8', 'no') + assert not np.can_cast('i8', 'no') + + assert np.can_cast('i8', 'equiv') + assert not np.can_cast('i8', 'equiv') + + assert np.can_cast('i8', 'safe') + assert not np.can_cast('i4', 'safe') + + assert np.can_cast('i4', 'same_kind') + assert not np.can_cast('u4', 'same_kind') + + assert np.can_cast('u4', 'unsafe') + + assert np.can_cast('bool', 'S5') + assert not np.can_cast('bool', 'S4') + + assert np.can_cast('b', 'S4') + assert not np.can_cast('b', 'S3') + + assert np.can_cast('u1', 'S3') + assert not np.can_cast('u1', 'S2') + assert np.can_cast('u2', 'S5') + assert not np.can_cast('u2', 'S4') + assert np.can_cast('u4', 'S10') + assert not np.can_cast('u4', 'S9') + assert np.can_cast('u8', 'S20') + assert not np.can_cast('u8', 'S19') + + assert np.can_cast('i1', 'S4') + assert not np.can_cast('i1', 'S3') + assert np.can_cast('i2', 'S6') + assert not np.can_cast('i2', 'S5') + assert np.can_cast('i4', 'S11') + assert not np.can_cast('i4', 'S10') + assert np.can_cast('i8', 'S21') + assert not np.can_cast('i8', 'S20') + + assert np.can_cast('bool', 'S5') + assert not np.can_cast('bool', 'S4') + + assert np.can_cast('b', 'U4') + assert not np.can_cast('b', 'U3') + + assert np.can_cast('u1', 'U3') + assert not np.can_cast('u1', 'U2') + assert np.can_cast('u2', 'U5') + assert not np.can_cast('u2', 'U4') + assert np.can_cast('u4', 'U10') + assert not np.can_cast('u4', 'U9') + assert np.can_cast('u8', 'U20') + assert not np.can_cast('u8', 'U19') + + assert np.can_cast('i1', 'U4') + assert not np.can_cast('i1', 'U3') + assert np.can_cast('i2', 'U6') + assert not np.can_cast('i2', 'U5') + assert np.can_cast('i4', 'U11') + assert not np.can_cast('i4', 'U10') + assert np.can_cast('i8', 'U21') + assert not np.can_cast('i8', 'U20') + + raises(TypeError, np.can_cast, 'i4', None) + raises(TypeError, np.can_cast, None, 'i4') + From noreply at buildbot.pypy.org Thu Apr 23 09:57:21 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 23 Apr 2015 09:57:21 +0200 (CEST) Subject: [pypy-commit] pypy object-dtype2: add upstream compatible logic to allow True for 'foo' != np.arange(3)[1] in a shorcut hack Message-ID: <20150423075721.64E951C12C8@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: object-dtype2 Changeset: r76898:cbba3abb9864 Date: 2015-04-23 10:52 +0300 http://bitbucket.org/pypy/pypy/changeset/cbba3abb9864/ Log: add upstream compatible logic to allow True for 'foo' != np.arange(3)[1] in a shorcut hack diff --git a/pypy/module/micronumpy/test/test_object_arrays.py b/pypy/module/micronumpy/test/test_object_arrays.py --- a/pypy/module/micronumpy/test/test_object_arrays.py +++ b/pypy/module/micronumpy/test/test_object_arrays.py @@ -37,6 +37,11 @@ assert (b == a).all() c = np.array([1, 2, 3]) assert (a[0] != c[0]) + assert (c[0] != a[0]) + assert (a[0] > c[0]) + assert (not a[0] < c[0]) + assert (c[0] < a[0]) + assert (not c[0] > a[0]) def test_logical_ufunc(self): import numpy as np diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py --- a/pypy/module/micronumpy/ufuncs.py +++ b/pypy/module/micronumpy/ufuncs.py @@ -441,8 +441,15 @@ elif w_ldtype.is_str() and w_rdtype.is_str() and \ self.comparison_func: pass - elif (w_ldtype.is_str() or w_rdtype.is_str()) and \ + elif (w_ldtype.is_str()) and \ self.comparison_func and w_out is None: + if self.name in ('equal', 'less_equal', 'less'): + return space.wrap(False) + return space.wrap(True) + elif (w_rdtype.is_str()) and \ + self.comparison_func and w_out is None: + if self.name in ('not_equal','less', 'less_equal'): + return space.wrap(True) return space.wrap(False) elif w_ldtype.is_flexible() or w_rdtype.is_flexible(): if self.comparison_func: From noreply at buildbot.pypy.org Thu Apr 23 18:14:04 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Thu, 23 Apr 2015 18:14:04 +0200 (CEST) Subject: [pypy-commit] pypy default: hack to support pytest 2.7 Message-ID: <20150423161404.4ECF11C0755@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76899:a24deef350ce Date: 2015-04-23 17:13 +0100 http://bitbucket.org/pypy/pypy/changeset/a24deef350ce/ Log: hack to support pytest 2.7 diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py --- a/pypy/tool/pytest/appsupport.py +++ b/pypy/tool/pytest/appsupport.py @@ -99,6 +99,7 @@ debug_excs = getattr(operr, 'debug_excs', []) if debug_excs: self._excinfo = debug_excs[0] + self.value = self.operr.errorstr(self.space) # XXX def __repr__(self): return "" % self.operr.errorstr(self.space) From noreply at buildbot.pypy.org Thu Apr 23 19:03:08 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 23 Apr 2015 19:03:08 +0200 (CEST) Subject: [pypy-commit] pypy object-dtype2: adjust tests to pass -A, leave fixing pypy to after branch merge Message-ID: <20150423170308.F04511C12C8@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: object-dtype2 Changeset: r76900:7f4e8ce39e3a Date: 2015-04-23 18:41 +0300 http://bitbucket.org/pypy/pypy/changeset/7f4e8ce39e3a/ Log: adjust tests to pass -A, leave fixing pypy to after branch merge diff --git a/pypy/module/micronumpy/test/test_object_arrays.py b/pypy/module/micronumpy/test/test_object_arrays.py --- a/pypy/module/micronumpy/test/test_object_arrays.py +++ b/pypy/module/micronumpy/test/test_object_arrays.py @@ -45,6 +45,10 @@ def test_logical_ufunc(self): import numpy as np + import sys + + if '__pypy__' in sys.builtin_module_names: + skip('need to refactor use of raw_xxx_op in types to make this work') a = np.array(["foo"], dtype=object) b = np.array([1], dtype=object) d = np.array([complex(1, 10)], dtype=object) @@ -75,12 +79,15 @@ def test_complex_op(self): import numpy as np + import sys a = np.array(['abc', 'def'], dtype=object) b = np.array([1, 2, 3], dtype=object) c = np.array([complex(1, 1), complex(1, -1)], dtype=object) for arg in (a,b,c): assert (arg == np.real(arg)).all() assert (0 == np.imag(arg)).all() + if '__pypy__' in sys.builtin_module_names: + skip('not implemented yet') raises(AttributeError, np.conj, a) res = np.conj(b) assert (res == b).all() @@ -102,6 +109,7 @@ def test_array_interface(self): import numpy as np + import sys class DummyArray(object): def __init__(self, interface, base=None): self.__array_interface__ = interface @@ -111,6 +119,8 @@ interface = dict(a.__array_interface__) interface['shape'] = tuple([3]) interface['strides'] = tuple([0]) + if '__pypy__' in sys.builtin_module_names: + skip('not implemented yet') c = np.array(DummyArray(interface, base=a)) c.dtype = a.dtype #print c @@ -118,6 +128,7 @@ def test_for_object_scalar_creation(self): import numpy as np + import sys a = np.object_() b = np.object_(3) b2 = np.object_(3.0) @@ -129,6 +140,8 @@ assert type(c) is np.ndarray assert c.dtype == object assert type(d) is type(None) + if '__pypy__' in sys.builtin_module_names: + skip('not implemented yet') e = np.object_([None, {}, []]) assert e.dtype == object @@ -137,36 +150,13 @@ # test segfaulted during ObjectType.store due to # missing gc hooks import numpy as np - dt = np.dtype([('x', int), ('y', np.object_)]) + import sys + ytype = np.object_ + if '__pypy__' in sys.builtin_module_names: + ytype = str + dt = np.dtype([('x', int), ('y', ytype)]) # Correct way a = np.array([(1, 'object')], dt) # Wrong way - should complain about writing buffer to object dtype - raises(np.array, [1, 'object'], dt) + raises(ValueError, np.array, [1, 'object'], dt) - def test_zeros(self): - skip('move this to unicode test when we have them') - import numpy as np - def sixu(s): - return unicode(s, 'unicode_escape') - - def buffer_len(arr): - if isinstance(arr, np.ndarray): - return len(arr.data) - return len(buffer(arr)) - - def content_check(ua, ua_scalar, nbtyes): - assert int(ua.dtype.str[2:]) == ulen - assert buffer_len(ua) == 4*ulen - assert ua_scalar -- sixu('') - assert ua_scalar.encode('ascii') == '' - assert buffer_len(ua_scalar) == 0 - - for ulen in [1, 2, 1099]: - ua = np.zeros((), dtype='U%s' % ulen) - content_check(ua, ua[()], 4 * ulen) - ua = zeros((2,), dtype='U%s' % ulen) - content_check(ua, ua[0], 4 * ulen*2) - content_check(ua, ua[1], 4 * ulen*2) - ua = zeros((2, 3, 4), dtype='U%s' % ulen) - content_check(ua, ua[0, 0, 0], 4 * ulen * 2 * 3 * 4) - content_check(ua, ua[-1, -1, -1], 4 * ulen * 2 * 3 * 4) From noreply at buildbot.pypy.org Thu Apr 23 19:03:10 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 23 Apr 2015 19:03:10 +0200 (CEST) Subject: [pypy-commit] pypy default: fix/update whatsnew-head with an eye towards PyPy 2.6 Message-ID: <20150423170310.298D01C12C8@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76901:3d4aa967c95c Date: 2015-04-23 19:57 +0300 http://bitbucket.org/pypy/pypy/changeset/3d4aa967c95c/ Log: fix/update whatsnew-head with an eye towards PyPy 2.6 diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,12 +5,46 @@ .. this is a revision shortly after release-2.5.1 .. startrev: cb01edcb59414d9d93056e54ed060673d24e67c1 -Issue #2017: on non-Linux-x86 platforms, reduced the memory impact of +issue2005: +ignore errors on closing random file handles while importing a module (cpython compatibility) + +issue2013: +added constants to _ssl for TLS 1.1 and 1.2 + +issue2014: +Add PyLong_FromUnicode to cpyext. + +issue2017: +On non-Linux-x86 platforms, reduced the memory impact of creating a lot of greenlets/tasklets. Particularly useful on Win32 and on ARM, where you used to get a MemoryError after only 2500-5000 greenlets (the 32-bit address space is exhausted). +Update gdb_pypy for python3 (gdb comatability) + +Merged rstrategies into rpython which provides a library for Storage Strategies + +Support unicode strings in numpy.dtype creation i.e. np.dtype(u'int64') + +Various rpython cleanups for vmprof support + +issue2019: +Fix isspace as called by rpython unicode.strip() + +issue2023: +In the cpyext 'Concrete Object Layer' API, +don't call methods on the object (which can be overriden), +but directly on the concrete base type. + +issue2029: +Hide the default_factory attribute in a dict + +issue2027: +Better document pyinteractive and add --withmod-time + .. branch: gc-incminimark-pinning-improve + +branch gc-incminimark-pinning-improve: Object Pinning is now used in `bz2` and `rzlib` (therefore also affects Python's `zlib`). In case the data to compress/decompress is inside the nursery (incminimark) it no longer needs to create a non-moving copy of it. This saves @@ -18,8 +52,15 @@ is introduced (`PYPY_GC_MAX_PINNED`) primarily for debugging purposes. .. branch: refactor-pycall + +branch refactor-pycall: Make `*`-unpacking in RPython function calls completely equivalent to passing the tuple's elements as arguments. In other words, `f(*(a, b))` now behaves exactly like `f(a, b)`. .. branch: issue2018 +branch issue2018: +Allow prebuilt rpython dict with function values + +.. branch: vmprof +.. Merged but then backed out, hopefully it will return as vmprof2 From noreply at buildbot.pypy.org Thu Apr 23 19:03:11 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 23 Apr 2015 19:03:11 +0200 (CEST) Subject: [pypy-commit] pypy object-dtype2: merge default into branch Message-ID: <20150423170311.9B1021C12C8@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: object-dtype2 Changeset: r76902:fcc3a72a167e Date: 2015-04-23 19:58 +0300 http://bitbucket.org/pypy/pypy/changeset/fcc3a72a167e/ Log: merge default into branch diff too long, truncating to 2000 out of 2315 lines diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,12 +5,46 @@ .. this is a revision shortly after release-2.5.1 .. startrev: cb01edcb59414d9d93056e54ed060673d24e67c1 -Issue #2017: on non-Linux-x86 platforms, reduced the memory impact of +issue2005: +ignore errors on closing random file handles while importing a module (cpython compatibility) + +issue2013: +added constants to _ssl for TLS 1.1 and 1.2 + +issue2014: +Add PyLong_FromUnicode to cpyext. + +issue2017: +On non-Linux-x86 platforms, reduced the memory impact of creating a lot of greenlets/tasklets. Particularly useful on Win32 and on ARM, where you used to get a MemoryError after only 2500-5000 greenlets (the 32-bit address space is exhausted). +Update gdb_pypy for python3 (gdb comatability) + +Merged rstrategies into rpython which provides a library for Storage Strategies + +Support unicode strings in numpy.dtype creation i.e. np.dtype(u'int64') + +Various rpython cleanups for vmprof support + +issue2019: +Fix isspace as called by rpython unicode.strip() + +issue2023: +In the cpyext 'Concrete Object Layer' API, +don't call methods on the object (which can be overriden), +but directly on the concrete base type. + +issue2029: +Hide the default_factory attribute in a dict + +issue2027: +Better document pyinteractive and add --withmod-time + .. branch: gc-incminimark-pinning-improve + +branch gc-incminimark-pinning-improve: Object Pinning is now used in `bz2` and `rzlib` (therefore also affects Python's `zlib`). In case the data to compress/decompress is inside the nursery (incminimark) it no longer needs to create a non-moving copy of it. This saves @@ -18,8 +52,15 @@ is introduced (`PYPY_GC_MAX_PINNED`) primarily for debugging purposes. .. branch: refactor-pycall + +branch refactor-pycall: Make `*`-unpacking in RPython function calls completely equivalent to passing the tuple's elements as arguments. In other words, `f(*(a, b))` now behaves exactly like `f(a, b)`. .. branch: issue2018 +branch issue2018: +Allow prebuilt rpython dict with function values + +.. branch: vmprof +.. Merged but then backed out, hopefully it will return as vmprof2 diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py --- a/pypy/module/micronumpy/descriptor.py +++ b/pypy/module/micronumpy/descriptor.py @@ -559,9 +559,11 @@ if space.is_none(w_dtype): return cache.w_float64dtype - elif space.isinstance_w(w_dtype, w_subtype): + if space.isinstance_w(w_dtype, w_subtype): return w_dtype - elif space.isinstance_w(w_dtype, space.w_str): + if space.isinstance_w(w_dtype, space.w_unicode): + w_dtype = space.wrap(space.str_w(w_dtype)) # may raise if invalid + if space.isinstance_w(w_dtype, space.w_str): name = space.str_w(w_dtype) if _check_for_commastring(name): return dtype_from_spec(space, w_dtype) diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -58,6 +58,7 @@ assert exc.value[0] == "there are no fields defined" assert dtype('int8').num == 1 + assert dtype(u'int8').num == 1 assert dtype('int8').name == 'int8' assert dtype('void').name == 'void' assert dtype(int).fields is None diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py --- a/pypy/tool/pytest/appsupport.py +++ b/pypy/tool/pytest/appsupport.py @@ -99,6 +99,7 @@ debug_excs = getattr(operr, 'debug_excs', []) if debug_excs: self._excinfo = debug_excs[0] + self.value = self.operr.errorstr(self.space) # XXX def __repr__(self): return "" % self.operr.errorstr(self.space) diff --git a/rpython/doc/rlib.rst b/rpython/doc/rlib.rst --- a/rpython/doc/rlib.rst +++ b/rpython/doc/rlib.rst @@ -128,6 +128,14 @@ a hierarchy of Address classes, in a typical static-OO-programming style. +rstrategies +----------- + +The :source:`rpython/rlib/rstrategies` module contains a library to implement storage strategies in +RPython VMs. The library is language-independent and extensible. +More details and examples can be found in the :doc:`rstrategies documentation `. + + streamio -------- diff --git a/rpython/doc/rstrategies.rst b/rpython/doc/rstrategies.rst new file mode 100644 --- /dev/null +++ b/rpython/doc/rstrategies.rst @@ -0,0 +1,209 @@ +rstrategies +=========== + +A library to implement storage strategies in VMs based on the RPython +toolchain. rstrategies can be used in VMs for any language or language +family. + +This library has been developed as part of a Masters Thesis by `Anton +Gulenko `__. + +The original paper describing the optimization "Storage Strategies for +collections in dynamically typed languages" by C.F. Bolz, L. Diekmann +and L. Tratt can be found +`here `__. + +So far, this library has been adpoted by 3 VMs: +`RSqueak `__, +`Topaz `__ (`Forked +here `__) and +`Pycket `__ (`Forked +here `__). + +Concept +------- + +Collections are often used homogeneously, i.e. they contain only objects +of the same type. Primitive numeric types like ints or floats are +especially interesting for optimization. These cases can be optimized by +storing the unboxed data of these objects in consecutive memory. This is +done by letting a special "strategy" object handle the entire storage of +a collection. The collection object holds two separate references: one +to its strategy and one to its storage. Every operation on the +collection is delegated to the strategy, which accesses the storage when +needed. The strategy can be switched to a more suitable one, which might +require converting the storage array. + +Usage +~~~~~ + +The following are the steps needed to integrated rstrategies in an +RPython VM. Because of the special nature of this library it is not +enough to simply call some API methods; the library must be integrated +within existing VM classes using a metaclass, mixins and other +meta-programming techniques. + +The sequence of steps described here is something like a "setup +walkthrough", and might be a bit abstract. To see a concrete example, +look at +`SingletonStorageStrategy `__, +`StrategyFactory `__ +and +`W\_PointersObject `__ +from the `RSqueak VM `__. The +code is also well commented. + +Basics +------- + +Currently the rstrategies library supports fixed sized and variable +sized collections. This can be used to optimize a wide range of +primitive data structures like arrays, lists or regular objects. Any of +these are called 'collections' in this context. The VM should have a +central class or class hierarchy for collections. In order to extend +these classes and use strategies, the library needs accessor methods for +two attributes of collection objects: strategy and storage. The easiest +way is adding the following line to the body of the root collection +class: + +:: + + rstrategies.make_accessors(strategy='strategy', storage='storage') + +This will generate the 4 accessor methods +``_[get/set]_[storage/strategy]()`` for the respective attributes. +Alternatively, implement these methods manually or overwrite the +getters/setters in ``StrategyFactory``. + +Next, the strategy classes must be defined. This requires a small class +hierarchy with a dedicated root class. In the definition of this root +class, include the following lines: + +:: + + __metaclass__ = rstrategies.StrategyMetaclass + import_from_mixin(rstrategies.AbstractStrategy) + import_from_mixin(rstrategies.SafeIndexingMixin) + +``import_from_mixin`` can be found in ``rpython.rlib.objectmodel``. If +index-checking is performed safely at other places in the VM, you can +use ``rstrategies.UnsafeIndexingMixin`` instead. If you need your own +metaclass, you can combine yours with the rstrategies one using multiple +inheritance `like +here `__. +Also implement a ``storage_factory()`` method, which returns an instance +of ``rstrategies.StorageFactory``, which is described below. + +An example ``AbstractStrategy`` class, which also stores an additional ``space`` parameter could looks like this: + +:: + + class AbstractStrategy(AbstractStrategy): + _attrs_ = ['space'] + _immutable_fields_ = ['space'] + __metaclass__ = rstrat.StrategyMetaclass + import_from_mixin(rstrat.AbstractStrategy) + import_from_mixin(rstrategies.SafeIndexingMixin) + + def __init__(self, space): + self.space = space + + def strategy_factory(self): + return self.space.strategy_factory + + +Strategy classes +---------------- + +Now you can create the actual strategy classes, subclassing them from +the single root class. The following list summarizes the basic +strategies available. + +- ``EmptyStrategy`` A strategy for empty collections; very efficient, but limited. Does not allocate anything. +- ``SingleValueStrategy`` A strategy for collections containing the same object ``n`` times. Only allocates memory to store the size of the collection. +- ``GenericStrategy`` A non-optimized strategy backed by a generic python list. This is the fallback strategy, since it can store everything, but is not optimized. +- ``WeakGenericStrategy`` Like ``GenericStrategy``, but uses ``weakref`` to hold on weakly to its elements. +- ``SingleTypeStrategy`` Can store a single unboxed type like int or float. This is the main optimizing strategy +- ``TaggingStrategy`` Extension of SingleTypeStrategy. Uses a specific value in the value range of the unboxed type to represent one additional, arbitrary object. For example, one of ``float``'s ``NaN`` representations can be used to represent special value like ``nil``. + +There are also intermediate classes, which allow creating new, more +customized strategies. For this, you should get familiar with the code. + +Include one of these mixin classes using ``import_from_mixin``. The +mixin classes contain comments describing methods or fields which are +also required in the strategy class in order to use them. Additionally, +add the ``@rstrategies.strategy(generalize=alist)`` decorator to all +strategy classes. The ``alist`` parameter must contain all strategies, +which the decorated strategy can switch to, if it can not represent a +new element anymore. +`Example `__ +for an implemented strategy. See the other strategy classes behind this +link for more examples. + +An example strategy class for optimized ``int`` storage could look like this: + +:: + + @rstrat.strategy(generalize=[GenericStrategy]) + class IntegerOrNilStrategy(AbstractStrategy): + import_from_mixin(rstrat.TaggingStrategy) + contained_type = model.W_Integer + def wrap(self, val): return self.space.wrap_int(val) + def unwrap(self, w_val): return self.space.unwrap_int(w_val) + def wrapped_tagged_value(self): return self.space.w_nil + def unwrapped_tagged_value(self): return constants.MAXINT + +Strategy Factory +---------------- + +The last part is subclassing ``rstrategies.StrategyFactory``, +overwriting the method ``instantiate_strategy`` if necessary and passing +the strategies root class to the constructor. The factory provides the +methods ``switch_strategy``, ``set_initial_strategy``, +``strategy_type_for`` which can be used by the VM code to use the +mechanism behind strategies. See the comments in the source code. + +The strategy mixins offer the following methods to manipulate the +contents of the collection: + +- basic API + + - ``size`` + +- fixed size API + + - ``store``, ``fetch``, ``slice``, ``store_all``, ``fetch_all`` + +- variable size API + + - ``insert``, ``delete``, ``append``, ``pop`` + +If the collection has a fixed size, simply never use any of the variable +size methods in the VM code. Since the strategies are singletons, these +methods need the collection object as first parameter. For convenience, +more fitting accessor methods should be implemented on the collection +class itself. + +An example strategy factory for the ``AbstractStrategy`` class above could look like this: + +:: + + class StrategyFactory(rstrategies.StrategyFactory): + _attrs_ = ['space'] + _immutable_fields_ = ['space'] + + def __init__(self, space): + self.space = space + rstrat.StrategyFactory.__init__(self, AbstractStrategy) + + def instantiate_strategy(self, strategy_type): + return strategy_type(self.space) + + def strategy_type_for(self, list_w, weak=False): + """ + Helper method for handling weak objects specially + """ + if weak: + return WeakListStrategy + return rstrategies.StrategyFactory.strategy_type_for(self, list_w) + \ No newline at end of file diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -62,6 +62,10 @@ op = operations[i] if op.getopnum() == rop.DEBUG_MERGE_POINT: continue + # ---------- GETFIELD_GC ---------- + if op.getopnum() == rop.GETFIELD_GC: + self.handle_getfield_gc(op) + continue # ---------- turn NEWxxx into CALL_MALLOC_xxx ---------- if op.is_malloc(): self.handle_malloc_operation(op) @@ -122,6 +126,18 @@ # ---------- + def handle_getfield_gc(self, op): + """See test_zero_ptr_field_before_getfield(). We hope there is + no getfield_gc in the middle of initialization code, but there + shouldn't be, given that a 'new' is already delayed by previous + optimization steps. In practice it should immediately be + followed by a bunch of 'setfields', and the 'pending_zeros' + optimization we do here is meant for this case.""" + self.emit_pending_zeros() + self.newops.append(op) + + # ---------- + def handle_malloc_operation(self, op): opnum = op.getopnum() if opnum == rop.NEW: diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py --- a/rpython/jit/backend/llsupport/test/test_rewrite.py +++ b/rpython/jit/backend/llsupport/test/test_rewrite.py @@ -1031,3 +1031,21 @@ guard_false(i1, descr=guarddescr) [] jump() """) + + def test_zero_ptr_field_before_getfield(self): + # This case may need to be fixed in the metainterp/optimizeopt + # already so that it no longer occurs for rewrite.py. But anyway + # it's a good idea to make sure rewrite.py is correct on its own. + self.check_rewrite(""" + [] + p0 = new(descr=tdescr) + p1 = getfield_gc(p0, descr=tdescr) + jump(p1) + """, """ + [] + p0 = call_malloc_nursery(%(tdescr.size)d) + setfield_gc(p0, 5678, descr=tiddescr) + zero_ptr_field(p0, %(tdescr.gc_fielddescrs[0].offset)s) + p1 = getfield_gc(p0, descr=tdescr) + jump(p1) + """) diff --git a/rpython/rlib/rstrategies/.coveragerc b/rpython/rlib/rstrategies/.coveragerc new file mode 100644 --- /dev/null +++ b/rpython/rlib/rstrategies/.coveragerc @@ -0,0 +1,10 @@ +# .coveragerc file to control coverage.py (code coverage plugin for pytest) +# Get it here: https://pypi.python.org/pypi/pytest-cov +# Examples: +# $ python -m pytest test --cov rpython.rlib.rstrategies --cov-report html --cov-config .coveragerc + +[run] +omit = + test/* + */__init__.py + logparser.py diff --git a/rpython/rlib/rstrategies/__init__.py b/rpython/rlib/rstrategies/__init__.py new file mode 100644 --- /dev/null +++ b/rpython/rlib/rstrategies/__init__.py @@ -0,0 +1,1 @@ +# Empy diff --git a/rpython/rlib/rstrategies/logger.py b/rpython/rlib/rstrategies/logger.py new file mode 100644 --- /dev/null +++ b/rpython/rlib/rstrategies/logger.py @@ -0,0 +1,58 @@ + +class LogEntry(object): + def __init__(self): + self.slots = 0 + self.objects = 0 + self.element_typenames = {} + + def add(self, size, element_typename): + self.slots += size + self.objects += 1 + if element_typename: + self.element_typenames[element_typename] = None + + def classnames(self): + return self.element_typenames.keys() + +class Logger(object): + _attrs_ = ["active", "aggregate", "logs"] + _immutable_fields_ = ["active?", "aggregate?", "logs"] + + def __init__(self): + self.active = False + self.aggregate = False + self.logs = {} + + def activate(self, aggregate=False): + self.active = True + self.aggregate = self.aggregate or aggregate + + def log(self, new_strategy, size, cause="", old_strategy="", typename="", element_typename=""): + if self.aggregate: + key = (cause, old_strategy, new_strategy, typename) + if key not in self.logs: + self.logs[key] = LogEntry() + entry = self.logs[key] + entry.add(size, element_typename) + else: + element_typenames = [ element_typename ] if element_typename else [] + self.output(cause, old_strategy, new_strategy, typename, size, 1, element_typenames) + + def print_aggregated_log(self): + if not self.aggregate: + return + for key, entry in self.logs.items(): + cause, old_strategy, new_strategy, typename = key + slots, objects, element_typenames = entry.slots, entry.objects, entry.classnames() + self.output(cause, old_strategy, new_strategy, typename, slots, objects, element_typenames) + + def output(self, cause, old_strategy, new_strategy, typename, slots, objects, element_typenames): + old_strategy_string = "%s -> " % old_strategy if old_strategy else "" + classname_string = " of %s" % typename if typename else "" + element_string = (" elements: " + " ".join(element_typenames)) if element_typenames else "" + format = (cause, old_strategy_string, new_strategy, classname_string, slots, objects, element_string) + self.do_print("%s (%s%s)%s size %d objects %d%s" % format) + + def do_print(self, str): + # Hook to increase testability + print str diff --git a/rpython/rlib/rstrategies/logparser.py b/rpython/rlib/rstrategies/logparser.py new file mode 100644 --- /dev/null +++ b/rpython/rlib/rstrategies/logparser.py @@ -0,0 +1,694 @@ + +import re, os, sys, operator + +""" +This script parses a log produced by rstrategies_logger.py into a graph and converts it to various outputs. +The most useful outputs are the dot* commands producing a visualization of the log using the dot-command of graphviz. +Every strategy is a node in the graph, and the edges are collections or objects that transition between +two strategies at some point during the log. +Artificial nodes are created for log entries without an explicit source node. These are the events when a +collection is created. +The input to this script is a logfile, a command and optional flags. +If the name of the logfile includes one of the AVAILABLE_VMS as a substring, the first three global variables +are automatically configured. +The script should work without these configurations, but the output will probably not be that pretty. +To avoid errors, the -a flag is implied when running without proper configuration. +""" + +# This should contain a full list of storage nodes (strategies). +# All strategies not included here will be combined into a single "Other"-node, if the -a flag is not given. +STORAGE_NODES = [] + +# This allows arbitrary renamings of storage strategy nodes +NODE_RENAMINGS = {} + +# Artificial storage-source nodes are automatically named like the associated operation. +# This dict allows customizing the names of these nodes. +STORAGE_SOURCES = {} + +def SET_VM(vm_name): + global STORAGE_NODES + global NODE_RENAMINGS + global STORAGE_SOURCES + if vm_name == 'RSqueak': + STORAGE_NODES = ['List', 'WeakList', 'SmallIntegerOrNil', 'FloatOrNil', 'AllNil'] + NODE_RENAMINGS = dict((x+'Strategy', x) for x in STORAGE_NODES) + STORAGE_SOURCES = {'Filledin': 'Image Loading', 'Initialized': 'Object Creation'} + elif vm_name == 'Pycket': + STORAGE_SOURCES = {'Created': 'Array Creation'} + # TODO + elif vm_name == 'Topaz': + # TODO + pass + else: + raise Exception("Unhandled vm name %s" % vm_name) + +AVAILABLE_VMS = ['RSqueak', 'Pycket', 'Topaz'] + +def configure_vm(logfile, flags): + vm_config_name = None + for vm_name in AVAILABLE_VMS: + if vm_name in logfile: + vm_config_name = vm_name + break + if vm_config_name is not None: + print "Using VM configuration %s" % vm_name + SET_VM(vm_name) + else: + print "No VM configuration found in filename '%s'. Available configurations: %s" % \ + (logfile, AVAILABLE_VMS) + print "Please add new VM configuration or rename logfile. Turning on -a flag to avoid errors." + flags.allstorage = True + +# ==================================================================== +# ======== Logfile parsing +# ==================================================================== + +def percent(part, total): + if total == 0: + return 0 + return float(part)*100 / total + +def parse(filename, flags, callback): + parsed_entries = 0 + if filename == "-": + opener = lambda: sys.stdin + else: + opener = lambda: open(filename, 'r', 1) + with opener() as file: + while True: + line = file.readline() + if len(line) == 0: + break + entry = parse_line(line, flags) + if entry: + parsed_entries += 1 + callback(entry) + return parsed_entries + +line_pattern = re.compile("^(?P\w+) $((?P\w+) -> )?(?P\w+)$( of (?P.+))? size (?P[0-9]+)( objects (?P[0-9]+))?( elements: (?P.+( .+)*))?$") + +def parse_line(line, flags): + result = line_pattern.match(line) + if result is None: + if flags.verbose: + print "Could not parse line: %s" % line[:-1] + return None + operation = str(result.group('operation')) + old_storage = result.group('old') + new_storage = str(result.group('new')) + classname = str(result.group('classname')) + size = int(result.group('size')) + objects = result.group('objects') + objects = int(objects) if objects else 1 + classnames = result.group('classnames') + if classnames is not None: + classnames = classnames.split(' ') + classnames = set(classnames) + else: + classnames = set() + + is_storage_source = old_storage is None + if is_storage_source: + if operation in STORAGE_SOURCES: + old_storage = STORAGE_SOURCES[operation] + else: + print "Using operation %s as storage source." % operation + old_storage = str(old_storage) + + if new_storage in NODE_RENAMINGS: + new_storage = NODE_RENAMINGS[new_storage] + if old_storage in NODE_RENAMINGS: + old_storage = NODE_RENAMINGS[old_storage] + + return LogEntry(operation, old_storage, new_storage, classname, size, objects, classnames, is_storage_source) + +class LogEntry(object): + + def __init__(self, operation, old_storage, new_storage, classname, size, objects, classnames, is_storage_source): + self.operation = operation + self.old_storage = old_storage + self.new_storage = new_storage + self.classname = classname + self.size = size + self.objects = objects + self.classnames = classnames + self.is_storage_source = is_storage_source + assert old_storage != new_storage, "old and new storage identical in log entry: %s" % self + + def full_key(self): + return (self.operation, self.old_storage, self.new_storage) + + def __lt__(self, other): + return self.classname < other.classname + + def __repr__(self): + return "%s(%s)" % (self.__str__(), object.__repr__(self)) + + def __str__(self): + old_storage_string = "%s -> " % self.old_storage if self.old_storage else "" + classname_string = " of %s" % self.classname if self.classname else "" + objects_string = " objects %d" % self.objects if self.objects > 1 else "" + return "%s (%s%s)%s size %d%s" % (self.operation, old_storage_string, self.new_storage, classname_string, self.size, objects_string) + +# ==================================================================== +# ======== Graph parsing +# ==================================================================== + +class Operations(object): + + def __init__(self, objects=0, slots=0, element_classnames=[]): + self.objects = objects + self.slots = slots + self.element_classnames = set(element_classnames) + + def __str__(self, total=None): + if self.objects == 0: + avg_slots = 0 + else: + avg_slots = float(self.slots) / self.objects + if total is not None and total.slots != 0: + percent_slots = " (%.1f%%)" % percent(self.slots, total.slots) + else: + percent_slots = "" + if total is not None and total.objects != 0: + percent_objects = " (%.1f%%)" % percent(self.objects, total.objects) + else: + percent_objects = "" + slots = format(self.slots, ",d") + objects = format(self.objects, ",d") + classnames = (" [ elements: %s ]" % ' '.join([str(x) for x in self.element_classnames])) \ + if len(self.element_classnames) else "" + return "%s%s slots in %s%s objects (avg size: %.1f)%s" % (slots, percent_slots, objects, percent_objects, avg_slots, classnames) + + def __repr__(self): + return "%s(%s)" % (self.__str__(), object.__repr__(self)) + + def add_log_entry(self, entry): + self.slots = self.slots + entry.size + self.objects = self.objects + entry.objects + self.element_classnames |= entry.classnames + + def __sub__(self, other): + return Operations(self.objects - other.objects, self.slots - other.slots) + + def __add__(self, other): + return Operations(self.objects + other.objects, self.slots + other.slots) + + def __lt__(self, other): + return self.slots < other.slots + + def empty(self): + return self.objects == 0 and self.slots == 0 + + def prefixprint(self, key="", total=None): + if not self.empty(): + print "%s%s" % (key, self.__str__(total)) + +class ClassOperations(object): + + def __init__(self): + self.classes = {} + + def cls(self, name): + if name not in self.classes: + self.classes[name] = Operations() + return self.classes[name] + + def total(self): + return reduce(operator.add, self.classes.values(), Operations()) + + def __str__(self): + return "ClassOperations(%s)" % self.classes + + def __repr__(self): + return "%s(%s)" % (self.__str__(), object.__repr__(self)) + + def __add__(self, other): + result = ClassOperations() + result.classes = dict(self.classes) + for classname, other_class in other.classes.items(): + result.cls(classname) # Make sure exists. + result.classes[classname] += other_class + return result + + def __sub__(self, other): + result = ClassOperations() + result.classes = dict(self.classes) + for classname, other_class in other.classes.items(): + result.cls(classname) # Make sure exists. + result.classes[classname] -= other_class + return result + +class StorageEdge(object): + + def __init__(self, operation="None", origin=None, target=None): + self.operation = operation + self.classes = ClassOperations() + self.origin = origin + self.target = target + self.is_storage_source = False + + def full_key(self): + return (self.operation, self.origin.name, self.target.name) + + def cls(self, classname): + return self.classes.cls(classname) + + def total(self): + return self.classes.total() + + def notify_nodes(self): + self.origin.note_outgoing(self) + self.target.note_incoming(self) + + def add_log_entry(self, entry): + self.cls(entry.classname).add_log_entry(entry) + if entry.is_storage_source: + self.is_storage_source = True + + def as_log_entries(self): + entries = [] + for classname, ops in self.classes.classes.items(): + origin = None if self.is_storage_source else self.origin.name + entry = LogEntry(self.operation, origin, self.target.name, classname, + ops.slots, ops.objects, ops.element_classnames, self.is_storage_source) + entries.append(entry) + return entries + + def __lt__(self, other): + return self.full_key() < other.full_key() + + def __str__(self): + return "[%s %s -> %s]" % (self.operation, self.origin, self.target) + + def __repr__(self): + return "%s(%s)" % (self.__str__(), object.__repr__(self)) + + def __add__(self, other): + origin = self.origin if self.origin is not None else other.origin + target = self.target if self.target is not None else other.target + result = StorageEdge(self.operation, origin, target) + result.classes += self.classes + other.classes + return result + + def __sub__(self, other): + origin = self.origin if self.origin is not None else other.origin + target = self.target if self.target is not None else other.target + result = StorageEdge(self.operation, origin, target) + result.classes += self.classes - other.classes + return result + +class StorageNode(object): + + def __init__(self, name): + self.name = name + self.incoming = set() + self.outgoing = set() + + def note_incoming(self, edge): + assert edge.target is self + if edge not in self.incoming: + self.incoming.add(edge) + + def note_outgoing(self, edge): + assert edge.origin is self + if edge not in self.outgoing: + self.outgoing.add(edge) + + def incoming_edges(self, operation): + return filter(lambda x: x.operation == operation, self.incoming) + + def outgoing_edges(self, operation): + return filter(lambda x: x.operation == operation, self.outgoing) + + def sum_incoming(self, operation): + return reduce(operator.add, self.incoming_edges(operation), StorageEdge(operation)) + + def sum_outgoing(self, operation): + return reduce(operator.add, self.outgoing_edges(operation), StorageEdge(operation)) + + def sum_all_incoming(self): + return reduce(operator.add, self.incoming, StorageEdge()) + + def sum_all_outgoing(self): + return reduce(operator.add, self.outgoing, StorageEdge()) + + def __str__(self): + return self.name + + def __repr__(self): + return "%s(%s)" % (self.__str__(), object.__repr__(self)) + + def merge_edge_sets(self, set1, set2, key_slot): + getter = lambda edge: edge.__dict__[key_slot] + set_dict = dict([(getter(edge), edge) for edge in set1]) + for edge in set2: + key = getter(edge) + if key not in set_dict: + set_dict[key] = edge + else: + set_dict[key] += edge + return set(set_dict.values()) + + def __add__(self, other): + result = StorageNode("%s %s" % (self.name, other.name)) + result.incoming = self.merge_edge_sets(self.incoming, other.incoming, "origin") + # TODO bad code + for edge in result.incoming: + edge.target = result + result.outgoing = self.merge_edge_sets(self.outgoing, other.outgoing, "target") + for edge in result.outgoing: + edge.origin = result + return result + + def __lt__(self, other): + return self.name < other.name + + def is_artificial(self): + for outgoing in self.outgoing: + if outgoing.is_storage_source: + return True + return False + + def is_storage_node(self): + return self.is_artificial() or self.name in STORAGE_NODES + + def dot_name(self): + return self.name.replace(" ", "_") + +class StorageGraph(object): + + def __init__(self): + self.nodes = {} + self.edges = {} + self.operations = set() + + def node(self, name): + if name not in self.nodes: + self.nodes[name] = StorageNode(name) + return self.nodes[name] + + def assert_sanity(self): + visited_edges = set() + for node in self.nodes.values(): + for edge in node.incoming: + assert edge in self.edges.values(), "Edge not in graph's edges: %s" % edge + visited_edges.add(edge) + if not edge.target is node: + print "Wrong edge target: %s\nIncoming edge: %s\nIn node: %s" % (edge.target, edge, node) + assert False + if not edge in edge.origin.outgoing: + print "Edge not in origin's outgoing: %s\nIncoming edge: %s\nIn node: %s" % (edge.origin.outgoing, edge, node) + assert False + for edge in node.outgoing: + assert edge in self.edges.values(), "Edge not in graph's edges: %s" % edge + visited_edges.add(edge) + if not edge.origin is node: + print "Wrong edge origin: %s\nOutgoing edge: %s\nIn node: %s" % (edge.origin, edge, node) + assert False + if not edge in edge.target.incoming: + print "Edge not in origin's incoming: %s\nOutgoing edge: %s\nIn node: %s" % (edge.target.incoming, edge, node) + assert False + assert len(visited_edges) == len(self.edges.values()), "Not all of graph's edges visited." + + def add_log_entry(self, log_entry): + self.operations.add(log_entry.operation) + key = log_entry.full_key() + if key not in self.edges: + edge = StorageEdge(log_entry.operation, self.node(log_entry.old_storage), self.node(log_entry.new_storage)) + self.edges[key] = edge + edge.notify_nodes() + self.edges[key].add_log_entry(log_entry) + + def collapse_nodes(self, collapsed_nodes, new_name=None): + if len(collapsed_nodes) == 0: + return + for node in collapsed_nodes: + del self.nodes[node.name] + for edge in node.incoming: + del self.edges[edge.full_key()] + for edge in node.outgoing: + del self.edges[edge.full_key()] + new_node = reduce(operator.add, collapsed_nodes) + if new_name is not None: + new_node.name = new_name + self.nodes[new_node.name] = new_node + # TODO bad code + for node in collapsed_nodes: + for edge in node.incoming: + edge.origin.outgoing.remove(edge) + new_edges = filter(lambda filtered: filtered.origin == edge.origin, new_node.incoming) + assert len(new_edges) == 1 + edge.origin.outgoing.add(new_edges[0]) + for edge in node.outgoing: + edge.target.incoming.remove(edge) + new_edges = filter(lambda filtered: filtered.target == edge.target, new_node.outgoing) + assert len(new_edges) == 1 + edge.target.incoming.add(new_edges[0]) + for edge in new_node.incoming: + self.edges[edge.full_key()] = edge + for edge in new_node.outgoing: + self.edges[edge.full_key()] = edge + self.assert_sanity() + + def collapse_nonstorage_nodes(self, new_name=None): + nodes = filter(lambda x: not x.is_storage_node(), self.nodes.values()) + self.collapse_nodes(nodes, new_name) + + def sorted_nodes(self): + nodes = self.nodes.values() + nodes.sort() + return nodes + +def make_graph(logfile, flags): + graph = StorageGraph() + def callback(entry): + graph.add_log_entry(entry) + parse(logfile, flags, callback) + graph.assert_sanity() + return graph + +# ==================================================================== +# ======== Command - Summarize log content +# ==================================================================== + +def command_summarize(logfile, flags): + graph = make_graph(logfile, flags) + if not flags.allstorage: + graph.collapse_nonstorage_nodes() + for node in graph.sorted_nodes(): + node.print_summary(flags, graph.operations) + +def StorageNode_print_summary(self, flags, all_operations): + print "\n%s:" % self.name + sum = StorageEdge() + total_incoming = self.sum_all_incoming().total() if flags.percent else None + + print "\tIncoming:" + for operation in all_operations: + if flags.detailed: + edges = [ (edge.origin.name, edge) for edge in self.incoming_edges(operation) ] + else: + edges = [ (operation, self.sum_incoming(operation)) ] + for edgename, edge in edges: + edge.print_with_name("\t\t\t", edgename, total_incoming, flags) + sum += edge + + print "\tOutgoing:" + for operation in all_operations: + if flags.detailed: + edges = [ (edge.target.name, edge) for edge in self.outgoing_edges(operation) ] + else: + edges = [ (operation, self.sum_outgoing(operation)) ] + for edgename, edge in edges: + edge.print_with_name("\t\t\t", edgename, total_incoming, flags) + sum -= edge + + sum.print_with_name("\t", "Remaining", total_incoming, flags) + +StorageNode.print_summary = StorageNode_print_summary + +def StorageEdge_print_with_name(self, prefix, edgename, total_reference, flags): + if flags.classes: + print "%s%s:" % (prefix, edgename) + prefix += "\t\t" + operations = self.classes.classes.items() + operations.sort(reverse=True, key=operator.itemgetter(1)) + else: + operations = [ (edgename, self.total()) ] + for classname, classops in operations: + classops.prefixprint("%s%s: " % (prefix, classname), total_reference) + +StorageEdge.print_with_name = StorageEdge_print_with_name + +# ==================================================================== +# ======== Command - DOT output +# ==================================================================== + +# Output is valid dot code and can be parsed by the graphviz dot utility. +def command_print_dot(logfile, flags): + graph = make_graph(logfile, flags) + print "/*" + print "Storage Statistics (dot format):" + print "================================" + print "*/" + print dot_string(graph, flags) + +def run_dot(logfile, flags, output_type): + import subprocess + dot = dot_string(make_graph(logfile, flags), flags) + command = ["dot", "-T%s" % output_type, "-o%s.%s" % (flags.logfile, output_type)] + print "Running:\n%s" % " ".join(command) + p = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + output = p.communicate(input=dot)[0] + print output + +def command_dot(logfile, flags): + run_dot(logfile, flags, "jpg") +def command_dot_ps(logfile, flags): + run_dot(logfile, flags, "ps") +def command_dot_pdf(logfile, flags): + run_dot(logfile, flags, "pdf") +def command_dot_svg(logfile, flags): + run_dot(logfile, flags, "svg") + +def dot_string(graph, flags): + result = "digraph G {" + incoming_cache = {} + if not flags.allstorage: + graph.collapse_nonstorage_nodes("Other") + + def make_label(edge, prefix="", total_edge=None, slots_per_object=False): + object_suffix = " objects" + slots_suffix = " slots" + if not flags.objects or not flags.slots: + object_suffix = slots_suffix = "" + if total_edge and flags.percent and total_edge.objects != 0: + percent_objects = " (%.1f%%)" % percent(edge.objects, total_edge.objects) + percent_slots = " (%.1f%%)" % percent(edge.slots, total_edge.slots) + else: + percent_objects = percent_slots = "" + label = "" + if flags.objects: + label += "%s%s%s%s
" % (prefix, format(edge.objects, ",.0f"), object_suffix, percent_objects) + if flags.slots: + label += "%s%s%s%s
" % (prefix, format(edge.slots, ",.0f"), slots_suffix, percent_slots) + if slots_per_object and flags.slotsPerObject: + label += "%.1f slots/object
" % (float(total.slots) / total.objects) + return label + + for node in graph.nodes.values(): + incoming = node.sum_all_incoming().total() + outgoing = node.sum_all_outgoing().total() + remaining = incoming - outgoing + if node.is_artificial(): + incoming_cache[node.name] = outgoing + shape = ",shape=box" + label = make_label(outgoing) + else: + incoming_cache[node.name] = incoming + shape = "" + label = make_label(incoming, "Incoming: ") + if remaining.objects != incoming.objects: + label += make_label(remaining, "Remaining: ", incoming) + result += "%s [label=<%s
%s>%s];" % (node.dot_name(), node.name, label, shape) + + for edge in graph.edges.values(): + total = edge.total() + incoming = incoming_cache[edge.origin.name] + label = make_label(total, "", incoming, slots_per_object=True) + target_node = edge.target.dot_name() + source_node = edge.origin.dot_name() + result += "%s -> %s [label=<%s>];" % (source_node, target_node, label) + + result += "}" + return result + +# ==================================================================== +# ======== Other commands +# ==================================================================== + +def command_aggregate(logfile, flags): + graph = make_graph(logfile, flags) + edges = graph.edges.values() + edges.sort() + for edge in edges: + logentries = edge.as_log_entries() + logentries.sort() + for entry in logentries: + print entry + +def command_print_entries(logfile, flags): + def callback(entry): + print entry + parse(logfile, flags, callback) + +# ==================================================================== +# ======== Main +# ==================================================================== + +class Flags(object): + + def __init__(self, flags): + self.flags = {} + for name, short in flags: + self.__dict__[name] = False + self.flags[short] = name + + def handle(self, arg): + if arg in self.flags: + self.__dict__[self.flags[arg]] = True + return True + else: + return False + + def __str__(self): + descriptions = [ ("%s (%s)" % description) for description in self.flags.items() ] + return "[%s]" % " | ".join(descriptions) + +def usage(flags, commands): + print "Arguments: logfile command %s" % flags + print "Available commands: %s" % commands + exit(1) + +def main(argv): + flags = Flags([ + # General + ('verbose', '-v'), + + # All outputs + ('percent', '-p'), + ('allstorage', '-a'), + + # Text outputs + ('detailed', '-d'), + ('classes', '-c'), + + # dot outputs + ('slots', '-s'), + ('objects', '-o'), + ('slotsPerObject', '-S'), + ]) + + command_prefix = "command_" + module = sys.modules[__name__].__dict__ + commands = [ a[len(command_prefix):] for a in module.keys() if a.startswith(command_prefix) ] + + if len(argv) < 2: + usage(flags, commands) + logfile = argv[0] + flags.logfile = logfile + configure_vm(logfile, flags) + command = argv[1] + for flag in argv[2:]: + if not flags.handle(flag): + usage(flags, commands) + if command not in commands: + usage(flags, commands) + + func = module[command_prefix + command] + func(logfile, flags) + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/rpython/rlib/rstrategies/rstrategies.py b/rpython/rlib/rstrategies/rstrategies.py new file mode 100644 --- /dev/null +++ b/rpython/rlib/rstrategies/rstrategies.py @@ -0,0 +1,572 @@ + +import weakref, sys +from rpython.rlib.rstrategies import logger +from rpython.rlib import jit, objectmodel, rerased +from rpython.rlib.objectmodel import specialize + +def make_accessors(strategy='strategy', storage='storage'): + """ + Instead of using this generator, the methods can be implemented manually. + A third way is to overwrite the getter/setter methods in StrategyFactory. + """ + def make_getter(attr): + def getter(self): return getattr(self, attr) + return getter + def make_setter(attr): + def setter(self, val): setattr(self, attr, val) + return setter + classdef = sys._getframe(1).f_locals + classdef['_get_strategy'] = make_getter(strategy) + classdef['_set_strategy'] = make_setter(strategy) + classdef['_get_storage'] = make_getter(storage) + classdef['_set_storage'] = make_setter(storage) + +class StrategyMetaclass(type): + """ + A metaclass is required, because we need certain attributes to be special + for every single strategy class. + """ + def __new__(self, name, bases, attrs): + attrs['_is_strategy'] = False + attrs['_is_singleton'] = False + attrs['_specializations'] = [] + # Not every strategy uses rerased-pairs, but they won't hurt + erase, unerase = rerased.new_erasing_pair(name) + def get_storage(self, w_self): + erased = self.strategy_factory().get_storage(w_self) + return unerase(erased) + def set_storage(self, w_self, storage): + erased = erase(storage) + self.strategy_factory().set_storage(w_self, erased) + attrs['get_storage'] = get_storage + attrs['set_storage'] = set_storage + return type.__new__(self, name, bases, attrs) + +def strategy(generalize=None, singleton=True): + """ + Strategy classes must be decorated with this. + generalize is a list of other strategies, that can be switched to from the decorated strategy. + If the singleton flag is set to False, new strategy instances will be created, + instead of always reusing the singleton object. + """ + def decorator(strategy_class): + # Patch strategy class: Add generalized_strategy_for and mark as strategy class. + if generalize: + @jit.unroll_safe + def generalized_strategy_for(self, value): + # TODO - optimize this method + for strategy in generalize: + if self.strategy_factory().strategy_singleton_instance(strategy)._check_can_handle(value): + return strategy + raise Exception("Could not find generalized strategy for %s coming from %s" % (value, self)) + strategy_class.generalized_strategy_for = generalized_strategy_for + for generalized in generalize: + generalized._specializations.append(strategy_class) + strategy_class._is_strategy = True + strategy_class._generalizations = generalize + strategy_class._is_singleton = singleton + return strategy_class + return decorator + +class StrategyFactory(object): + _immutable_fields_ = ["strategies[*]", "logger", "strategy_singleton_field"] + factory_instance_counter = 0 + + def __init__(self, root_class, all_strategy_classes=None): + if all_strategy_classes is None: + all_strategy_classes = self._collect_subclasses(root_class) + self.strategies = [] + self.logger = logger.Logger() + + # This is to avoid confusion between multiple factories existing simultaneously (e.g. in tests) + self.strategy_singleton_field = "__singleton_%i" % StrategyFactory.factory_instance_counter + StrategyFactory.factory_instance_counter += 1 + + self._create_strategy_instances(root_class, all_strategy_classes) + + def _create_strategy_instances(self, root_class, all_strategy_classes): + for strategy_class in all_strategy_classes: + if strategy_class._is_strategy: + setattr(strategy_class, self.strategy_singleton_field, self.instantiate_strategy(strategy_class)) + self.strategies.append(strategy_class) + self._patch_strategy_class(strategy_class, root_class) + self._order_strategies() + + # ============================= + # API methods + # ============================= + + def switch_strategy(self, w_self, new_strategy_type, new_element=None): + """ + Switch the strategy of w_self to the new type. + new_element can be given as as hint, purely for logging purposes. + It should be the object that was added to w_self, causing the strategy switch. + """ + old_strategy = self.get_strategy(w_self) + if new_strategy_type._is_singleton: + new_strategy = self.strategy_singleton_instance(new_strategy_type) + else: + size = old_strategy.size(w_self) + new_strategy = self.instantiate_strategy(new_strategy_type, w_self, size) + self.set_strategy(w_self, new_strategy) + old_strategy._convert_storage_to(w_self, new_strategy) + new_strategy.strategy_switched(w_self) + self.log(w_self, new_strategy, old_strategy, new_element) + return new_strategy + + def set_initial_strategy(self, w_self, strategy_type, size, elements=None): + """ + Initialize the strategy and storage fields of w_self. + This must be called before switch_strategy or any strategy method can be used. + elements is an optional list of values initially stored in w_self. + If given, then len(elements) == size must hold. + """ + assert self.get_strategy(w_self) is None, "Strategy should not be initialized yet!" + if strategy_type._is_singleton: + strategy = self.strategy_singleton_instance(strategy_type) + else: + strategy = self.instantiate_strategy(strategy_type, w_self, size) + self.set_strategy(w_self, strategy) + strategy._initialize_storage(w_self, size) + element = None + if elements: + strategy.store_all(w_self, elements) + if len(elements) > 0: element = elements[0] + strategy.strategy_switched(w_self) + self.log(w_self, strategy, None, element) + return strategy + + @jit.unroll_safe + def strategy_type_for(self, objects): + """ + Return the best-fitting strategy to hold all given objects. + """ + specialized_strategies = len(self.strategies) + can_handle = [True] * specialized_strategies + for obj in objects: + if specialized_strategies <= 1: + break + for i, strategy in enumerate(self.strategies): + if can_handle[i] and not self.strategy_singleton_instance(strategy)._check_can_handle(obj): + can_handle[i] = False + specialized_strategies -= 1 + for i, strategy_type in enumerate(self.strategies): + if can_handle[i]: + return strategy_type + raise Exception("Could not find strategy to handle: %s" % objects) + + def decorate_strategies(self, transitions): + """ + As an alternative to decorating all strategies with @strategy, + invoke this in the constructor of your StrategyFactory subclass, before + calling __init__. transitions is a dict mapping all strategy classes to + their 'generalize' list parameter (see @strategy decorator). + """ + "NOT_RPYTHON" + for strategy_class, generalized in transitions.items(): + strategy(generalized)(strategy_class) + + # ============================= + # The following methods can be overwritten to customize certain aspects of the factory. + # ============================= + + def instantiate_strategy(self, strategy_type, w_self=None, initial_size=0): + """ + Return a functional instance of strategy_type. + Overwrite this if you need a non-default constructor. + The two additional parameters should be ignored for singleton-strategies. + """ + return strategy_type() + + def log(self, w_self, new_strategy, old_strategy=None, new_element=None): + """ + This can be overwritten into a more appropriate call to self.logger.log + """ + if not self.logger.active: return + new_strategy_str = self.log_string_for_object(new_strategy) + old_strategy_str = self.log_string_for_object(old_strategy) + element_typename = self.log_string_for_object(new_element) + size = new_strategy.size(w_self) + typename = "" + cause = "Switched" if old_strategy else "Created" + self.logger.log(new_strategy_str, size, cause, old_strategy_str, typename, element_typename) + + @specialize.call_location() + def log_string_for_object(self, obj): + """ + This can be overwritten instead of the entire log() method. + Keep the specialize-annotation in order to handle different kinds of objects here. + """ + return obj.__class__.__name__ if obj else "" + + # These storage accessors are specialized because the storage field is + # populated by erased-objects which seem to be incompatible sometimes. + @specialize.call_location() + def get_storage(self, obj): + return obj._get_storage() + @specialize.call_location() + def set_storage(self, obj, val): + return obj._set_storage(val) + + def get_strategy(self, obj): + return obj._get_strategy() + def set_strategy(self, obj, val): + return obj._set_strategy(val) + + # ============================= + # Internal methods + # ============================= + + def _patch_strategy_class(self, strategy_class, root_class): + "NOT_RPYTHON" + # Patch root class: Add default handler for visitor + def _convert_storage_from_OTHER(self, w_self, previous_strategy): + self._convert_storage_from(w_self, previous_strategy) + funcname = "_convert_storage_from_" + strategy_class.__name__ + _convert_storage_from_OTHER.func_name = funcname + setattr(root_class, funcname, _convert_storage_from_OTHER) + + # Patch strategy class: Add polymorphic visitor function + def _convert_storage_to(self, w_self, new_strategy): + getattr(new_strategy, funcname)(w_self, self) + strategy_class._convert_storage_to = _convert_storage_to + + def _collect_subclasses(self, cls): + "NOT_RPYTHON" + subclasses = [] + for subcls in cls.__subclasses__(): + subclasses.append(subcls) + subclasses.extend(self._collect_subclasses(subcls)) + return subclasses + + def _order_strategies(self): + "NOT_RPYTHON" + def get_generalization_depth(strategy, visited=None): + if visited is None: + visited = set() + if strategy._generalizations: + if strategy in visited: + raise Exception("Cycle in generalization-tree of %s" % strategy) + visited.add(strategy) + depth = 0 + for generalization in strategy._generalizations: + other_depth = get_generalization_depth(generalization, set(visited)) + depth = max(depth, other_depth) + return depth + 1 + else: + return 0 + self.strategies.sort(key=get_generalization_depth, reverse=True) + + @jit.elidable + def strategy_singleton_instance(self, strategy_class): + return getattr(strategy_class, self.strategy_singleton_field) + + def _freeze_(self): + # Instance will be frozen at compile time, making accesses constant. + # The constructor does meta stuff which is not possible after translation. + return True + +class AbstractStrategy(object): + """ + == Required: + strategy_factory(self) - Access to StorageFactory + """ + + def strategy_switched(self, w_self): + # Overwrite this method for a hook whenever the strategy + # of w_self was switched to self. + pass + + # Main Fixedsize API + + def store(self, w_self, index0, value): + raise NotImplementedError("Abstract method") + + def fetch(self, w_self, index0): + raise NotImplementedError("Abstract method") + + def size(self, w_self): + raise NotImplementedError("Abstract method") + + # Fixedsize utility methods + + def slice(self, w_self, start, end): + return [ self.fetch(w_self, i) for i in range(start, end)] + + def fetch_all(self, w_self): + return self.slice(w_self, 0, self.size(w_self)) + + def store_all(self, w_self, elements): + for i, e in enumerate(elements): + self.store(w_self, i, e) + + # Main Varsize API + + def insert(self, w_self, index0, list_w): + raise NotImplementedError("Abstract method") + + def delete(self, w_self, start, end): + raise NotImplementedError("Abstract method") + + # Varsize utility methods + + def append(self, w_self, list_w): + self.insert(w_self, self.size(w_self), list_w) + + def pop(self, w_self, index0): + e = self.fetch(w_self, index0) + self.delete(w_self, index0, index0+1) + return e + + # Internal methods + + def _initialize_storage(self, w_self, initial_size): + raise NotImplementedError("Abstract method") + + def _check_can_handle(self, value): + raise NotImplementedError("Abstract method") + + def _convert_storage_to(self, w_self, new_strategy): + # This will be overwritten in _patch_strategy_class + new_strategy._convert_storage_from(w_self, self) + + @jit.unroll_safe + def _convert_storage_from(self, w_self, previous_strategy): + # This is a very unefficient (but most generic) way to do this. + # Subclasses should specialize. + storage = previous_strategy.fetch_all(w_self) + self._initialize_storage(w_self, previous_strategy.size(w_self)) + for i, field in enumerate(storage): + self.store(w_self, i, field) + + def _generalize_for_value(self, w_self, value): + strategy_type = self.generalized_strategy_for(value) + new_instance = self.strategy_factory().switch_strategy(w_self, strategy_type, new_element=value) + return new_instance + + def _cannot_handle_store(self, w_self, index0, value): + new_instance = self._generalize_for_value(w_self, value) + new_instance.store(w_self, index0, value) + + def _cannot_handle_insert(self, w_self, index0, list_w): + # TODO - optimize. Prevent multiple generalizations and slicing done by callers. + new_strategy = self._generalize_for_value(w_self, list_w[0]) + new_strategy.insert(w_self, index0, list_w) + +# ============== Special Strategies with no storage array ============== + +class EmptyStrategy(AbstractStrategy): + # == Required: + # See AbstractStrategy + + def _initialize_storage(self, w_self, initial_size): + assert initial_size == 0 + self.set_storage(w_self, None) + def _convert_storage_from(self, w_self, previous_strategy): + self.set_storage(w_self, None) + def _check_can_handle(self, value): + return False + + def fetch(self, w_self, index0): + raise IndexError + def store(self, w_self, index0, value): + self._cannot_handle_store(w_self, index0, [value]) + def insert(self, w_self, index0, list_w): + self._cannot_handle_insert(w_self, index0, list_w) + def delete(self, w_self, start, end): + self.check_index_range(w_self, start, end) + def size(self, w_self): + return 0 + +class SingleValueStrategyStorage(object): + """Small container object for a size value.""" + _attrs_ = ['size'] + def __init__(self, size=0): + self.size = size + +class SingleValueStrategy(AbstractStrategy): + # == Required: + # See AbstractStrategy + # check_index_*(...) - use mixin SafeIndexingMixin or UnsafeIndexingMixin + # value(self) - the single value contained in this strategy. Should be constant. + + def _initialize_storage(self, w_self, initial_size): + storage_obj = SingleValueStrategyStorage(initial_size) + self.set_storage(w_self, storage_obj) + def _convert_storage_from(self, w_self, previous_strategy): + self._initialize_storage(w_self, previous_strategy.size(w_self)) + def _check_can_handle(self, value): + return value is self.value() + + def fetch(self, w_self, index0): + self.check_index_fetch(w_self, index0) + return self.value() + def store(self, w_self, index0, value): + self.check_index_store(w_self, index0) + if self._check_can_handle(value): + return + self._cannot_handle_store(w_self, index0, value) + def delete(self, w_self, start, end): + self.check_index_range(w_self, start, end) + self.get_storage(w_self).size -= (end - start) + def size(self, w_self): + return self.get_storage(w_self).size + + @jit.unroll_safe + def insert(self, w_self, index0, list_w): + storage_obj = self.get_storage(w_self) + for i in range(len(list_w)): + if self._check_can_handle(list_w[i]): + storage_obj.size += 1 + else: + self._cannot_handle_insert(w_self, index0 + i, list_w[i:]) + return + +# ============== Basic strategies with storage ============== + +class StrategyWithStorage(AbstractStrategy): + # == Required: + # See AbstractStrategy + # check_index_*(...) - use mixin SafeIndexingMixin or UnsafeIndexingMixin + # default_value(self) - The value to be initially contained in this strategy + + def _initialize_storage(self, w_self, initial_size): + default = self._unwrap(self.default_value()) + self.set_storage(w_self, [default] * initial_size) + + @jit.unroll_safe + def _convert_storage_from(self, w_self, previous_strategy): + size = previous_strategy.size(w_self) + new_storage = [ self._unwrap(previous_strategy.fetch(w_self, i)) + for i in range(size) ] + self.set_storage(w_self, new_storage) + + def store(self, w_self, index0, wrapped_value): + self.check_index_store(w_self, index0) + if self._check_can_handle(wrapped_value): + unwrapped = self._unwrap(wrapped_value) + self.get_storage(w_self)[index0] = unwrapped + else: + self._cannot_handle_store(w_self, index0, wrapped_value) + + def fetch(self, w_self, index0): + self.check_index_fetch(w_self, index0) + unwrapped = self.get_storage(w_self)[index0] + return self._wrap(unwrapped) + + def _wrap(self, value): + raise NotImplementedError("Abstract method") + + def _unwrap(self, value): + raise NotImplementedError("Abstract method") + + def size(self, w_self): + return len(self.get_storage(w_self)) + + @jit.unroll_safe + def insert(self, w_self, start, list_w): + # This is following Python's behaviour - insert automatically + # happens at the beginning of an array, even if index is larger + if start > self.size(w_self): + start = self.size(w_self) + for i in range(len(list_w)): + if self._check_can_handle(list_w[i]): + self.get_storage(w_self).insert(start + i, self._unwrap(list_w[i])) + else: + self._cannot_handle_insert(w_self, start + i, list_w[i:]) + return + + def delete(self, w_self, start, end): + self.check_index_range(w_self, start, end) + assert start >= 0 and end >= 0 + del self.get_storage(w_self)[start : end] + +class GenericStrategy(StrategyWithStorage): + # == Required: + # See StrategyWithStorage + + def _wrap(self, value): + return value + def _unwrap(self, value): + return value + def _check_can_handle(self, wrapped_value): + return True + +class WeakGenericStrategy(StrategyWithStorage): + # == Required: + # See StrategyWithStorage + + def _wrap(self, value): + return value() or self.default_value() + def _unwrap(self, value): + assert value is not None + return weakref.ref(value) + def _check_can_handle(self, wrapped_value): + return True + +# ============== Mixins for index checking operations ============== + +class SafeIndexingMixin(object): + def check_index_store(self, w_self, index0): + self.check_index(w_self, index0) + def check_index_fetch(self, w_self, index0): + self.check_index(w_self, index0) + def check_index_range(self, w_self, start, end): + if end < start: + raise IndexError + self.check_index(w_self, start) + self.check_index(w_self, end) + def check_index(self, w_self, index0): + if index0 < 0 or index0 >= self.size(w_self): + raise IndexError + +class UnsafeIndexingMixin(object): + def check_index_store(self, w_self, index0): + pass + def check_index_fetch(self, w_self, index0): + pass + def check_index_range(self, w_self, start, end): + pass + +# ============== Specialized Storage Strategies ============== + +class SpecializedStrategy(StrategyWithStorage): + # == Required: + # See StrategyWithStorage + # wrap(self, value) - Return a boxed object for the primitive value + # unwrap(self, value) - Return the unboxed primitive value of value + + def _unwrap(self, value): + return self.unwrap(value) + def _wrap(self, value): + return self.wrap(value) + +class SingleTypeStrategy(SpecializedStrategy): + # == Required Functions: + # See SpecializedStrategy + # contained_type - The wrapped type that can be stored in this strategy + + def _check_can_handle(self, value): + return isinstance(value, self.contained_type) + +class TaggingStrategy(SingleTypeStrategy): + """This strategy uses a special tag value to represent a single additional object.""" + # == Required: + # See SingleTypeStrategy + # wrapped_tagged_value(self) - The tagged object + # unwrapped_tagged_value(self) - The unwrapped tag value representing the tagged object + + def _check_can_handle(self, value): + return value is self.wrapped_tagged_value() or \ + (isinstance(value, self.contained_type) and \ + self.unwrap(value) != self.unwrapped_tagged_value()) + + def _unwrap(self, value): + if value is self.wrapped_tagged_value(): + return self.unwrapped_tagged_value() + return self.unwrap(value) + + def _wrap(self, value): + if value == self.unwrapped_tagged_value(): + return self.wrapped_tagged_value() + return self.wrap(value) diff --git a/rpython/rlib/rstrategies/test/test_rstrategies.py b/rpython/rlib/rstrategies/test/test_rstrategies.py new file mode 100644 --- /dev/null +++ b/rpython/rlib/rstrategies/test/test_rstrategies.py @@ -0,0 +1,552 @@ + +import py +from rpython.rlib.rstrategies import rstrategies as rs +from rpython.rlib.objectmodel import import_from_mixin + +# === Define small model tree + +class W_AbstractObject(object): + pass + +class W_Object(W_AbstractObject): + pass + +class W_Integer(W_AbstractObject): + def __init__(self, value): + self.value = value + def __eq__(self, other): + return isinstance(other, W_Integer) and self.value == other.value + +class W_List(W_AbstractObject): + rs.make_accessors() + def __init__(self, strategy=None, size=0, elements=None): + self.strategy = None + if strategy: + factory.set_initial_strategy(self, strategy, size, elements) + def fetch(self, i): + assert self.strategy + return self.strategy.fetch(self, i) + def store(self, i, value): + assert self.strategy + return self.strategy.store(self, i, value) + def size(self): + assert self.strategy + return self.strategy.size(self) + def insert(self, index0, list_w): + assert self.strategy + return self.strategy.insert(self, index0, list_w) + def delete(self, start, end): + assert self.strategy + return self.strategy.delete(self, start, end) + def append(self, list_w): + assert self.strategy + return self.strategy.append(self, list_w) + def pop(self, index0): + assert self.strategy + return self.strategy.pop(self, index0) + def slice(self, start, end): + assert self.strategy + return self.strategy.slice(self, start, end) + def fetch_all(self): + assert self.strategy + return self.strategy.fetch_all(self) + def store_all(self, elements): + assert self.strategy + return self.strategy.store_all(self, elements) + +w_nil = W_Object() + +# === Define concrete strategy classes + +class AbstractStrategy(object): + __metaclass__ = rs.StrategyMetaclass + import_from_mixin(rs.AbstractStrategy) + import_from_mixin(rs.SafeIndexingMixin) + def __init__(self, factory, w_self=None, size=0): + self.factory = factory + def strategy_factory(self): + return self.factory + +class Factory(rs.StrategyFactory): + switching_log = [] + + def __init__(self, root_class): + self.decorate_strategies({ + EmptyStrategy: [NilStrategy, IntegerStrategy, IntegerOrNilStrategy, GenericStrategy], + NilStrategy: [IntegerOrNilStrategy, GenericStrategy], + GenericStrategy: [], + IntegerStrategy: [IntegerOrNilStrategy, GenericStrategy], + IntegerOrNilStrategy: [GenericStrategy], + }) + rs.StrategyFactory.__init__(self, root_class) + + def instantiate_strategy(self, strategy_type, w_self=None, size=0): + return strategy_type(self, w_self, size) + + def set_strategy(self, w_list, strategy): + old_strategy = self.get_strategy(w_list) + self.switching_log.append((old_strategy, strategy)) + super(Factory, self).set_strategy(w_list, strategy) + + def clear_log(self): + del self.switching_log[:] + +class EmptyStrategy(AbstractStrategy): + import_from_mixin(rs.EmptyStrategy) + # TODO - implement and test transition from Generic back to Empty + +class NilStrategy(AbstractStrategy): + import_from_mixin(rs.SingleValueStrategy) + def value(self): return w_nil + +class GenericStrategy(AbstractStrategy): + import_from_mixin(rs.GenericStrategy) + import_from_mixin(rs.UnsafeIndexingMixin) + def default_value(self): return w_nil + +class WeakGenericStrategy(AbstractStrategy): + import_from_mixin(rs.WeakGenericStrategy) + def default_value(self): return w_nil + +class IntegerStrategy(AbstractStrategy): + import_from_mixin(rs.SingleTypeStrategy) + contained_type = W_Integer + def wrap(self, value): return W_Integer(value) + def unwrap(self, value): return value.value + def default_value(self): return W_Integer(0) + +class IntegerOrNilStrategy(AbstractStrategy): + import_from_mixin(rs.TaggingStrategy) + contained_type = W_Integer + def wrap(self, value): return W_Integer(value) + def unwrap(self, value): return value.value + def default_value(self): return w_nil + def wrapped_tagged_value(self): return w_nil + def unwrapped_tagged_value(self): import sys; return sys.maxint + + at rs.strategy(generalize=[], singleton=False) +class NonSingletonStrategy(GenericStrategy): + def __init__(self, factory, w_list=None, size=0): + super(NonSingletonStrategy, self).__init__(factory, w_list, size) + self.w_list = w_list + self.the_size = size + +class NonStrategy(NonSingletonStrategy): + pass + + at rs.strategy(generalize=[]) +class InefficientStrategy(GenericStrategy): + def _convert_storage_from(self, w_self, previous_strategy): + return AbstractStrategy._convert_storage_from(self, w_self, previous_strategy) + +factory = Factory(AbstractStrategy) + +def check_contents(list, expected): + assert list.size() == len(expected) + for i, val in enumerate(expected): + assert list.fetch(i) == val + +def teardown(): + factory.clear_log() + +# === Test Initialization and fetch + +def test_setup(): + pass + +def test_factory_setup(): + expected_strategies = 7 + assert len(factory.strategies) == expected_strategies + assert len(set(factory.strategies)) == len(factory.strategies) + for strategy in factory.strategies: + assert isinstance(factory.strategy_singleton_instance(strategy), strategy) + +def test_factory_setup_singleton_instances(): + new_factory = Factory(AbstractStrategy) + s1 = factory.strategy_singleton_instance(GenericStrategy) + s2 = new_factory.strategy_singleton_instance(GenericStrategy) + assert s1 is not s2 + assert s1.strategy_factory() is factory + assert s2.strategy_factory() is new_factory + +def test_metaclass(): + assert NonStrategy._is_strategy == False + assert IntegerOrNilStrategy._is_strategy == True + assert IntegerOrNilStrategy._is_singleton == True + assert NonSingletonStrategy._is_singleton == False + assert NonStrategy._is_singleton == False + assert NonStrategy.get_storage is not NonSingletonStrategy.get_storage + +def test_singletons(): + def do_test_singletons(cls, expected_true): + l1 = W_List(cls, 0) + l2 = W_List(cls, 0) + if expected_true: + assert l1.strategy is l2.strategy + else: + assert l1.strategy is not l2.strategy + do_test_singletons(EmptyStrategy, True) + do_test_singletons(NonSingletonStrategy, False) + do_test_singletons(NonStrategy, False) + do_test_singletons(GenericStrategy, True) + +def do_test_initialization(cls, default_value=w_nil, is_safe=True): + size = 10 + l = W_List(cls, size) + s = l.strategy + assert s.size(l) == size + assert s.fetch(l,0) == default_value + assert s.fetch(l,size/2) == default_value + assert s.fetch(l,size-1) == default_value + py.test.raises(IndexError, s.fetch, l, size) + py.test.raises(IndexError, s.fetch, l, size+1) + py.test.raises(IndexError, s.fetch, l, size+5) + if is_safe: + py.test.raises(IndexError, s.fetch, l, -1) + else: + assert s.fetch(l, -1) == s.fetch(l, size - 1) + +def test_init_Empty(): + l = W_List(EmptyStrategy, 0) + s = l.strategy + assert s.size(l) == 0 + py.test.raises(IndexError, s.fetch, l, 0) + py.test.raises(IndexError, s.fetch, l, 10) + py.test.raises(IndexError, s.delete, l, 0, 1) + py.test.raises(AssertionError, W_List, EmptyStrategy, 2) # Only size 0 possible. + +def test_init_Nil(): + do_test_initialization(NilStrategy) + +def test_init_Generic(): + do_test_initialization(GenericStrategy, is_safe=False) + +def test_init_WeakGeneric(): + do_test_initialization(WeakGenericStrategy) + +def test_init_Integer(): + do_test_initialization(IntegerStrategy, default_value=W_Integer(0)) + +def test_init_IntegerOrNil(): + do_test_initialization(IntegerOrNilStrategy) + +# === Test Simple store + +def do_test_store(cls, stored_value=W_Object(), is_safe=True, is_varsize=False): + size = 10 + l = W_List(cls, size) + s = l.strategy From noreply at buildbot.pypy.org Thu Apr 23 19:03:12 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 23 Apr 2015 19:03:12 +0200 (CEST) Subject: [pypy-commit] pypy object-dtype2: document branch to be closed Message-ID: <20150423170312.C21D51C12C8@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: object-dtype2 Changeset: r76903:cc8c8b4aba30 Date: 2015-04-23 20:00 +0300 http://bitbucket.org/pypy/pypy/changeset/cc8c8b4aba30/ Log: document branch to be closed diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -64,3 +64,6 @@ .. branch: vmprof .. Merged but then backed out, hopefully it will return as vmprof2 + +.. branch: object-dtype2 +Extend numpy dtypes to allow using objects with associated garbage collection hook From noreply at buildbot.pypy.org Thu Apr 23 19:03:13 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 23 Apr 2015 19:03:13 +0200 (CEST) Subject: [pypy-commit] pypy object-dtype2: close branch to be merged Message-ID: <20150423170313.D9AC71C12C8@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: object-dtype2 Changeset: r76904:03977f2520fe Date: 2015-04-23 20:00 +0300 http://bitbucket.org/pypy/pypy/changeset/03977f2520fe/ Log: close branch to be merged From noreply at buildbot.pypy.org Thu Apr 23 19:03:15 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 23 Apr 2015 19:03:15 +0200 (CEST) Subject: [pypy-commit] pypy default: merge in object-dtype2, which provides an object dtype for numpy Message-ID: <20150423170315.7E3411C12C8@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76905:bc2c76a447dc Date: 2015-04-23 20:01 +0300 http://bitbucket.org/pypy/pypy/changeset/bc2c76a447dc/ Log: merge in object-dtype2, which provides an object dtype for numpy diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -64,3 +64,6 @@ .. branch: vmprof .. Merged but then backed out, hopefully it will return as vmprof2 + +.. branch: object-dtype2 +Extend numpy dtypes to allow using objects with associated garbage collection hook diff --git a/pypy/goal/targetnumpystandalone.py b/pypy/goal/targetnumpystandalone.py deleted file mode 100644 --- a/pypy/goal/targetnumpystandalone.py +++ /dev/null @@ -1,43 +0,0 @@ - -""" Usage: - -./targetnumpystandalone-c array_size - -Will execute a give numpy bytecode. Arrays will be ranges (in float) modulo 10, -constants would be consecutive starting from one. - -Bytecode should contain letters 'a' 'l' and 'f' so far and be correct -""" - -import time -from pypy.module.micronumpy.compile import numpy_compile -from rpython.jit.codewriter.policy import JitPolicy -from rpython.rtyper.annlowlevel import hlstr - -def entry_point(argv): - if len(argv) != 3: - print __doc__ - return 1 - try: - size = int(argv[2]) - except ValueError: - print "INVALID LITERAL FOR INT:", argv[2] - print __doc__ - return 3 - t0 = time.time() - main(argv[0], size) - print "bytecode:", argv[0], "size:", size - print "took:", time.time() - t0 - return 0 - -def main(bc, size): - if not isinstance(bc, str): - bc = hlstr(bc) # for tests - a = numpy_compile(bc, size) - a = a.compute() - -def target(*args): - return entry_point, None - -def jitpolicy(driver): - return JitPolicy() diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py --- a/pypy/module/micronumpy/__init__.py +++ b/pypy/module/micronumpy/__init__.py @@ -30,6 +30,9 @@ for c in ['MAXDIMS', 'CLIP', 'WRAP', 'RAISE']: interpleveldefs[c] = 'space.wrap(constants.%s)' % c + def startup(self, space): + from pypy.module.micronumpy.concrete import _setup + _setup() class UMathModule(MixedModule): appleveldefs = {} diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py --- a/pypy/module/micronumpy/base.py +++ b/pypy/module/micronumpy/base.py @@ -34,11 +34,13 @@ @staticmethod def from_shape(space, shape, dtype, order='C', w_instance=None, zero=True): - from pypy.module.micronumpy import concrete + from pypy.module.micronumpy import concrete, descriptor, boxes from pypy.module.micronumpy.strides import calc_strides strides, backstrides = calc_strides(shape, dtype.base, order) impl = concrete.ConcreteArray(shape, dtype.base, order, strides, backstrides, zero=zero) + if dtype == descriptor.get_dtype_cache(space).w_objectdtype: + impl.fill(space, boxes.W_ObjectBox(space.w_None)) if w_instance: return wrap_impl(space, space.type(w_instance), w_instance, impl) return W_NDimArray(impl) @@ -123,7 +125,7 @@ def get_shape(self): return self.implementation.get_shape() - def get_dtype(self): + def get_dtype(self, space=None): return self.implementation.dtype def get_order(self): diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py --- a/pypy/module/micronumpy/boxes.py +++ b/pypy/module/micronumpy/boxes.py @@ -607,6 +607,19 @@ # arr.storage[i] = arg[i] return W_UnicodeBox(arr, 0, arr.dtype) +class W_ObjectBox(W_GenericBox): + descr__new__, _get_dtype, descr_reduce = new_dtype_getter(NPY.OBJECT) + + def __init__(self, w_obj): + self.w_obj = w_obj + + def convert_to(self, space, dtype): + if dtype.is_bool(): + return W_BoolBox(space.bool_w(self.w_obj)) + return self # XXX + + def descr__getattr__(self, space, w_key): + return space.getattr(self.w_obj, w_key) W_GenericBox.typedef = TypeDef("numpy.generic", __new__ = interp2app(W_GenericBox.descr__new__.im_func), @@ -856,3 +869,9 @@ __new__ = interp2app(W_UnicodeBox.descr__new__unicode_box.im_func), __len__ = interp2app(W_UnicodeBox.descr_len), ) + +W_ObjectBox.typedef = TypeDef("numpy.object_", W_ObjectBox.typedef, + __new__ = interp2app(W_ObjectBox.descr__new__.im_func), + __getattr__ = interp2app(W_ObjectBox.descr__getattr__), +) + diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py --- a/pypy/module/micronumpy/compile.py +++ b/pypy/module/micronumpy/compile.py @@ -3,7 +3,7 @@ """ import re from pypy.interpreter import special -from pypy.interpreter.baseobjspace import InternalSpaceCache, W_Root +from pypy.interpreter.baseobjspace import InternalSpaceCache, W_Root, ObjSpace from pypy.interpreter.error import OperationError from rpython.rlib.objectmodel import specialize, instantiate from rpython.rlib.nonconst import NonConstant @@ -47,7 +47,7 @@ def lookup(self, name): return self.getdictvalue(self, name) -class FakeSpace(object): +class FakeSpace(ObjSpace): w_ValueError = W_TypeObject("ValueError") w_TypeError = W_TypeObject("TypeError") w_IndexError = W_TypeObject("IndexError") @@ -67,6 +67,7 @@ w_unicode = W_TypeObject("unicode") w_complex = W_TypeObject("complex") w_dict = W_TypeObject("dict") + w_object = W_TypeObject("object") def __init__(self): """NOT_RPYTHON""" @@ -88,7 +89,8 @@ return self.wrap(len(w_obj.items)) def getattr(self, w_obj, w_attr): - return StringObject(NonConstant('foo')) + assert isinstance(w_attr, StringObject) + return w_obj.getdictvalue(self, w_attr.v) def isinstance_w(self, w_obj, w_tp): try: diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py --- a/pypy/module/micronumpy/concrete.py +++ b/pypy/module/micronumpy/concrete.py @@ -1,11 +1,11 @@ from pypy.interpreter.error import OperationError, oefmt -from rpython.rlib import jit +from rpython.rlib import jit, rgc from rpython.rlib.buffer import Buffer -from rpython.rlib.debug import make_sure_not_resized +from rpython.rlib.debug import make_sure_not_resized, debug_print from rpython.rlib.rawstorage import alloc_raw_storage, free_raw_storage, \ raw_storage_getitem, raw_storage_setitem, RAW_STORAGE -from rpython.rtyper.lltypesystem import rffi, lltype -from pypy.module.micronumpy import support, loop +from rpython.rtyper.lltypesystem import rffi, lltype, llmemory +from pypy.module.micronumpy import support, loop, constants as NPY from pypy.module.micronumpy.base import convert_to_array, W_NDimArray, \ ArrayArgumentException from pypy.module.micronumpy.iterators import ArrayIter @@ -13,11 +13,13 @@ RecordChunk, calc_strides, calc_new_strides, shape_agreement, calculate_broadcast_strides, calc_backstrides) from rpython.rlib.objectmodel import keepalive_until_here +from rpython.rtyper.annlowlevel import cast_gcref_to_instance +from pypy.interpreter.baseobjspace import W_Root class BaseConcreteArray(object): _immutable_fields_ = ['dtype?', 'storage', 'start', 'size', 'shape[*]', - 'strides[*]', 'backstrides[*]', 'order'] + 'strides[*]', 'backstrides[*]', 'order', 'gcstruct'] start = 0 parent = None flags = 0 @@ -333,6 +335,44 @@ loop.setslice(space, impl.get_shape(), impl, self) return impl +OBJECTSTORE = lltype.GcStruct('ObjectStore', + ('length', lltype.Signed), + ('step', lltype.Signed), + ('storage', llmemory.Address), + rtti=True) +offset_of_storage = llmemory.offsetof(OBJECTSTORE, 'storage') +offset_of_length = llmemory.offsetof(OBJECTSTORE, 'length') +offset_of_step = llmemory.offsetof(OBJECTSTORE, 'step') + +V_OBJECTSTORE = lltype.nullptr(OBJECTSTORE) + +def customtrace(gc, obj, callback, arg): + #debug_print('in customtrace w/obj', obj) + length = (obj + offset_of_length).signed[0] + step = (obj + offset_of_step).signed[0] + storage = (obj + offset_of_storage).address[0] + #debug_print('tracing', length, 'objects in ndarray.storage') + i = 0 + while i < length: + gc._trace_callback(callback, arg, storage) + storage += step + i += 1 + +lambda_customtrace = lambda: customtrace + +def _setup(): + rgc.register_custom_trace_hook(OBJECTSTORE, lambda_customtrace) + + at jit.dont_look_inside +def _create_objectstore(storage, length, elsize): + gcstruct = lltype.malloc(OBJECTSTORE) + # JIT does not support cast_ptr_to_adr + gcstruct.storage = llmemory.cast_ptr_to_adr(storage) + #print 'create gcstruct',gcstruct,'with storage',storage,'as',gcstruct.storage + gcstruct.length = length + gcstruct.step = elsize + return gcstruct + class ConcreteArrayNotOwning(BaseConcreteArray): def __init__(self, shape, dtype, order, strides, backstrides, storage, start=0): @@ -347,10 +387,11 @@ self.backstrides = backstrides self.storage = storage self.start = start + self.gcstruct = V_OBJECTSTORE def fill(self, space, box): self.dtype.itemtype.fill(self.storage, self.dtype.elsize, - box, 0, self.size, 0) + box, 0, self.size, 0, self.gcstruct) def set_shape(self, space, orig_array, new_shape): strides, backstrides = calc_strides(new_shape, self.dtype, @@ -374,17 +415,24 @@ def base(self): return None - class ConcreteArray(ConcreteArrayNotOwning): def __init__(self, shape, dtype, order, strides, backstrides, storage=lltype.nullptr(RAW_STORAGE), zero=True): + gcstruct = V_OBJECTSTORE if storage == lltype.nullptr(RAW_STORAGE): - storage = dtype.itemtype.malloc(support.product(shape) * - dtype.elsize, zero=zero) + length = support.product(shape) + if dtype.num == NPY.OBJECT: + storage = dtype.itemtype.malloc(length * dtype.elsize, zero=True) + gcstruct = _create_objectstore(storage, length, dtype.elsize) + else: + storage = dtype.itemtype.malloc(length * dtype.elsize, zero=zero) ConcreteArrayNotOwning.__init__(self, shape, dtype, order, strides, backstrides, storage) + self.gcstruct = gcstruct def __del__(self): + if self.gcstruct: + self.gcstruct.length = 0 free_raw_storage(self.storage, track_allocation=False) @@ -423,6 +471,7 @@ parent = parent.parent # one level only self.parent = parent self.storage = parent.storage + self.gcstruct = parent.gcstruct self.order = parent.order self.dtype = dtype self.size = support.product(shape) * self.dtype.elsize @@ -480,6 +529,7 @@ class VoidBoxStorage(BaseConcreteArray): def __init__(self, size, dtype): self.storage = alloc_raw_storage(size) + self.gcstruct = V_OBJECTSTORE self.dtype = dtype self.size = size diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py --- a/pypy/module/micronumpy/ctors.py +++ b/pypy/module/micronumpy/ctors.py @@ -38,6 +38,34 @@ raise oefmt(space.w_ValueError, "object __array__ method not producing an array") +def try_interface_method(space, w_object): + try: + w_interface = space.getattr(w_object, space.wrap("__array_interface__")) + except OperationError, e: + if e.match(space, space.w_AttributeError): + return None + raise + if w_interface is None: + # happens from compile.py + return None + version = space.int_w(space.finditem(w_interface, space.wrap("version"))) + if version < 3: + raise oefmt(space.w_NotImplementedError, + "__array_interface__ version %d not supported", version) + # make a view into the data + w_shape = space.finditem(w_interface, space.wrap('shape')) + w_dtype = space.finditem(w_interface, space.wrap('typestr')) + w_descr = space.finditem(w_interface, space.wrap('descr')) + data_w = space.listview(space.finditem(w_interface, space.wrap('data'))) + w_strides = space.finditem(w_interface, space.wrap('strides')) + shape = [space.int_w(i) for i in space.listview(w_shape)] + dtype = descriptor.decode_w_dtype(space, w_dtype) + rw = space.is_true(data_w[1]) + #print 'create view from shape',shape,'dtype',dtype,'descr',w_descr,'data',data_w[0],'rw',rw + raise oefmt(space.w_NotImplementedError, + "creating array from __array_interface__ not supported yet") + return + @unwrap_spec(ndmin=int, copy=bool, subok=bool) def array(space, w_object, w_dtype=None, copy=True, w_order=None, subok=False, @@ -63,7 +91,11 @@ # continue with w_array, but do further operations in place w_object = w_array copy = False - + if not isinstance(w_object, W_NDimArray): + w_array = try_interface_method(space, w_object) + if w_array is not None: + w_object = w_array + copy = False dtype = descriptor.decode_w_dtype(space, w_dtype) if space.is_none(w_order): diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py --- a/pypy/module/micronumpy/descriptor.py +++ b/pypy/module/micronumpy/descriptor.py @@ -6,7 +6,7 @@ from pypy.interpreter.typedef import (TypeDef, GetSetProperty, interp_attrproperty, interp_attrproperty_w) from rpython.rlib import jit -from rpython.rlib.objectmodel import specialize, compute_hash +from rpython.rlib.objectmodel import specialize, compute_hash, we_are_translated from rpython.rlib.rarithmetic import r_longlong, r_ulonglong from pypy.module.micronumpy import types, boxes, base, support, constants as NPY from pypy.module.micronumpy.appbridge import get_appbridge_cache @@ -56,7 +56,7 @@ self.char = char self.w_box_type = w_box_type if byteorder is None: - if itemtype.get_element_size() == 1: + if itemtype.get_element_size() == 1 or isinstance(itemtype, types.ObjectType): byteorder = NPY.IGNORE else: byteorder = NPY.NATIVE @@ -112,6 +112,9 @@ def is_str(self): return self.num == NPY.STRING + def is_object(self): + return self.num == NPY.OBJECT + def is_str_or_unicode(self): return self.num == NPY.STRING or self.num == NPY.UNICODE @@ -428,7 +431,7 @@ self.names.append(name) self.fields[name] = offset, dtype - self.itemtype = types.RecordType() + self.itemtype = types.RecordType(space) if self.is_flexible(): self.elsize = size @@ -443,7 +446,7 @@ endian = NPY.OPPBYTE if self.is_native() else NPY.NATBYTE elif newendian != NPY.IGNORE: endian = newendian - itemtype = self.itemtype.__class__(endian in (NPY.NATIVE, NPY.NATBYTE)) + itemtype = self.itemtype.__class__(space, endian in (NPY.NATIVE, NPY.NATBYTE)) fields = self.fields if fields is None: fields = {} @@ -482,7 +485,7 @@ fields[fldname] = (offset, subdtype) offset += subdtype.elsize names.append(fldname) - return W_Dtype(types.RecordType(), NPY.VOID, NPY.VOIDLTR, NPY.VOIDLTR, + return W_Dtype(types.RecordType(space), NPY.VOID, NPY.VOIDLTR, NPY.VOIDLTR, space.gettypefor(boxes.W_VoidBox), names=names, fields=fields, elsize=offset) @@ -493,8 +496,17 @@ def dtype_from_spec(space, w_spec): - w_lst = get_appbridge_cache(space).call_method(space, - 'numpy.core._internal', '_commastring', Arguments(space, [w_spec])) + + if we_are_translated(): + w_lst = get_appbridge_cache(space).call_method(space, + 'numpy.core._internal', '_commastring', Arguments(space, [w_spec])) + else: + # testing, handle manually + if space.eq_w(w_spec, space.wrap('u4,u4,u4')): + w_lst = space.newlist([space.wrap('u4')]*3) + else: + raise oefmt(space.w_RuntimeError, + "cannot parse w_spec") if not space.isinstance_w(w_lst, space.w_list) or space.len_w(w_lst) < 1: raise oefmt(space.w_RuntimeError, "_commastring is not returning a list with len >= 1") @@ -541,7 +553,7 @@ if size == 1: return subdtype size *= subdtype.elsize - return W_Dtype(types.VoidType(), NPY.VOID, NPY.VOIDLTR, NPY.VOIDLTR, + return W_Dtype(types.VoidType(space), NPY.VOID, NPY.VOIDLTR, NPY.VOIDLTR, space.gettypefor(boxes.W_VoidBox), shape=shape, subdtype=subdtype, elsize=size) @@ -587,8 +599,7 @@ if w_dtype is dtype.w_box_type: return dtype if space.isinstance_w(w_dtype, space.w_type): - raise oefmt(space.w_NotImplementedError, - "cannot create dtype with type '%N'", w_dtype) + return cache.w_objectdtype raise oefmt(space.w_TypeError, "data type not understood") @@ -655,7 +666,7 @@ def new_string_dtype(space, size, char=NPY.STRINGLTR): return W_Dtype( - types.StringType(), + types.StringType(space), elsize=size, num=NPY.STRING, kind=NPY.STRINGLTR, @@ -665,7 +676,7 @@ def new_unicode_dtype(space, size): - itemtype = types.UnicodeType() + itemtype = types.UnicodeType(space) return W_Dtype( itemtype, elsize=size * itemtype.get_element_size(), @@ -678,7 +689,7 @@ def new_void_dtype(space, size): return W_Dtype( - types.VoidType(), + types.VoidType(space), elsize=size, num=NPY.VOID, kind=NPY.VOIDLTR, @@ -690,126 +701,126 @@ class DtypeCache(object): def __init__(self, space): self.w_booldtype = W_Dtype( - types.Bool(), + types.Bool(space), num=NPY.BOOL, kind=NPY.GENBOOLLTR, char=NPY.BOOLLTR, w_box_type=space.gettypefor(boxes.W_BoolBox), ) self.w_int8dtype = W_Dtype( - types.Int8(), + types.Int8(space), num=NPY.BYTE, kind=NPY.SIGNEDLTR, char=NPY.BYTELTR, w_box_type=space.gettypefor(boxes.W_Int8Box), ) self.w_uint8dtype = W_Dtype( - types.UInt8(), + types.UInt8(space), num=NPY.UBYTE, kind=NPY.UNSIGNEDLTR, char=NPY.UBYTELTR, w_box_type=space.gettypefor(boxes.W_UInt8Box), ) self.w_int16dtype = W_Dtype( - types.Int16(), + types.Int16(space), num=NPY.SHORT, kind=NPY.SIGNEDLTR, char=NPY.SHORTLTR, w_box_type=space.gettypefor(boxes.W_Int16Box), ) self.w_uint16dtype = W_Dtype( - types.UInt16(), + types.UInt16(space), num=NPY.USHORT, kind=NPY.UNSIGNEDLTR, char=NPY.USHORTLTR, w_box_type=space.gettypefor(boxes.W_UInt16Box), ) self.w_int32dtype = W_Dtype( - types.Int32(), + types.Int32(space), num=NPY.INT, kind=NPY.SIGNEDLTR, char=NPY.INTLTR, w_box_type=space.gettypefor(boxes.W_Int32Box), ) self.w_uint32dtype = W_Dtype( - types.UInt32(), + types.UInt32(space), num=NPY.UINT, kind=NPY.UNSIGNEDLTR, char=NPY.UINTLTR, w_box_type=space.gettypefor(boxes.W_UInt32Box), ) self.w_longdtype = W_Dtype( - types.Long(), + types.Long(space), num=NPY.LONG, kind=NPY.SIGNEDLTR, char=NPY.LONGLTR, w_box_type=space.gettypefor(boxes.W_LongBox), ) self.w_ulongdtype = W_Dtype( - types.ULong(), + types.ULong(space), num=NPY.ULONG, kind=NPY.UNSIGNEDLTR, char=NPY.ULONGLTR, w_box_type=space.gettypefor(boxes.W_ULongBox), ) self.w_int64dtype = W_Dtype( - types.Int64(), + types.Int64(space), num=NPY.LONGLONG, kind=NPY.SIGNEDLTR, char=NPY.LONGLONGLTR, w_box_type=space.gettypefor(boxes.W_Int64Box), ) self.w_uint64dtype = W_Dtype( - types.UInt64(), + types.UInt64(space), num=NPY.ULONGLONG, kind=NPY.UNSIGNEDLTR, char=NPY.ULONGLONGLTR, w_box_type=space.gettypefor(boxes.W_UInt64Box), ) self.w_float32dtype = W_Dtype( - types.Float32(), + types.Float32(space), num=NPY.FLOAT, kind=NPY.FLOATINGLTR, char=NPY.FLOATLTR, w_box_type=space.gettypefor(boxes.W_Float32Box), ) self.w_float64dtype = W_Dtype( - types.Float64(), + types.Float64(space), num=NPY.DOUBLE, kind=NPY.FLOATINGLTR, char=NPY.DOUBLELTR, w_box_type=space.gettypefor(boxes.W_Float64Box), ) self.w_floatlongdtype = W_Dtype( - types.FloatLong(), + types.FloatLong(space), num=NPY.LONGDOUBLE, kind=NPY.FLOATINGLTR, char=NPY.LONGDOUBLELTR, w_box_type=space.gettypefor(boxes.W_FloatLongBox), ) self.w_complex64dtype = W_Dtype( - types.Complex64(), + types.Complex64(space), num=NPY.CFLOAT, kind=NPY.COMPLEXLTR, char=NPY.CFLOATLTR, w_box_type=space.gettypefor(boxes.W_Complex64Box), ) self.w_complex128dtype = W_Dtype( - types.Complex128(), + types.Complex128(space), num=NPY.CDOUBLE, kind=NPY.COMPLEXLTR, char=NPY.CDOUBLELTR, w_box_type=space.gettypefor(boxes.W_Complex128Box), ) self.w_complexlongdtype = W_Dtype( - types.ComplexLong(), + types.ComplexLong(space), num=NPY.CLONGDOUBLE, kind=NPY.COMPLEXLTR, char=NPY.CLONGDOUBLELTR, w_box_type=space.gettypefor(boxes.W_ComplexLongBox), ) self.w_stringdtype = W_Dtype( - types.StringType(), + types.StringType(space), elsize=0, num=NPY.STRING, kind=NPY.STRINGLTR, @@ -817,7 +828,7 @@ w_box_type=space.gettypefor(boxes.W_StringBox), ) self.w_unicodedtype = W_Dtype( - types.UnicodeType(), + types.UnicodeType(space), elsize=0, num=NPY.UNICODE, kind=NPY.UNICODELTR, @@ -825,7 +836,7 @@ w_box_type=space.gettypefor(boxes.W_UnicodeBox), ) self.w_voiddtype = W_Dtype( - types.VoidType(), + types.VoidType(space), elsize=0, num=NPY.VOID, kind=NPY.VOIDLTR, @@ -833,26 +844,33 @@ w_box_type=space.gettypefor(boxes.W_VoidBox), ) self.w_float16dtype = W_Dtype( - types.Float16(), + types.Float16(space), num=NPY.HALF, kind=NPY.FLOATINGLTR, char=NPY.HALFLTR, w_box_type=space.gettypefor(boxes.W_Float16Box), ) self.w_intpdtype = W_Dtype( - types.Long(), + types.Long(space), num=NPY.LONG, kind=NPY.SIGNEDLTR, char=NPY.INTPLTR, w_box_type=space.gettypefor(boxes.W_LongBox), ) self.w_uintpdtype = W_Dtype( - types.ULong(), + types.ULong(space), num=NPY.ULONG, kind=NPY.UNSIGNEDLTR, char=NPY.UINTPLTR, w_box_type=space.gettypefor(boxes.W_ULongBox), ) + self.w_objectdtype = W_Dtype( + types.ObjectType(space), + num=NPY.OBJECT, + kind=NPY.OBJECTLTR, + char=NPY.OBJECTLTR, + w_box_type=space.gettypefor(boxes.W_ObjectBox), + ) aliases = { NPY.BOOL: ['bool_', 'bool8'], NPY.BYTE: ['byte'], @@ -871,6 +889,7 @@ NPY.CLONGDOUBLE: ['clongdouble', 'clongfloat'], NPY.STRING: ['string_', 'str'], NPY.UNICODE: ['unicode_'], + NPY.OBJECT: ['object_'], } self.alternate_constructors = { NPY.BOOL: [space.w_bool], @@ -889,6 +908,8 @@ NPY.UNICODE: [space.w_unicode], NPY.VOID: [space.gettypefor(boxes.W_GenericBox)], #space.w_buffer, # XXX no buffer in space + NPY.OBJECT: [space.gettypefor(boxes.W_ObjectBox), + space.w_object], } float_dtypes = [self.w_float16dtype, self.w_float32dtype, self.w_float64dtype, self.w_floatlongdtype] @@ -908,7 +929,7 @@ self.w_int64dtype, self.w_uint64dtype, ] + float_dtypes + complex_dtypes + [ self.w_stringdtype, self.w_unicodedtype, self.w_voiddtype, - self.w_intpdtype, self.w_uintpdtype, + self.w_intpdtype, self.w_uintpdtype, self.w_objectdtype, ] self.float_dtypes_by_num_bytes = sorted( (dtype.elsize, dtype) @@ -960,6 +981,7 @@ 'USHORT': self.w_uint16dtype, 'FLOAT': self.w_float32dtype, 'BOOL': self.w_booldtype, + 'OBJECT': self.w_objectdtype, } typeinfo_partial = { diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -202,11 +202,16 @@ return self elif isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool() \ and w_idx.ndims() > 0: - return self.getitem_filter(space, w_idx) - try: - return self.implementation.descr_getitem(space, self, w_idx) - except ArrayArgumentException: - return self.getitem_array_int(space, w_idx) + w_ret = self.getitem_filter(space, w_idx) + else: + try: + w_ret = self.implementation.descr_getitem(space, self, w_idx) + except ArrayArgumentException: + w_ret = self.getitem_array_int(space, w_idx) + if isinstance(w_ret, boxes.W_ObjectBox): + #return the W_Root object, not a scalar + w_ret = w_ret.w_obj + return w_ret def getitem(self, space, index_list): return self.implementation.getitem_index(space, index_list) @@ -550,6 +555,7 @@ else: strides = self.descr_get_strides(space) space.setitem_str(w_d, 'strides', strides) + space.setitem_str(w_d, 'version', space.wrap(3)) return w_d w_pypy_data = None @@ -845,7 +851,7 @@ "new type not compatible with array.")) # Strides, shape does not change v = impl.astype(space, dtype) - return wrap_impl(space, w_type, self, v) + return wrap_impl(space, w_type, self, v) strides = impl.get_strides() if dims == 1 or strides[0] = 0 - else: - a = array(Polynomial()) - assert a.shape == () + diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -17,6 +17,7 @@ def __init__(self): self.base = self self.elsize = 1 + self.num = 0 def create_slice(space, a, chunks): @@ -3150,11 +3151,7 @@ assert b[35] == 200 b[[slice(25, 30)]] = range(5) assert all(a[:5] == range(5)) - import sys - if '__pypy__' not in sys.builtin_module_names: - raises(TypeError, 'b[[[slice(25, 125)]]]') - else: - raises(NotImplementedError, 'b[[[slice(25, 125)]]]') + raises(IndexError, 'b[[[slice(25, 125)]]]') def test_cumsum(self): from numpy import arange diff --git a/pypy/module/micronumpy/test/test_object_arrays.py b/pypy/module/micronumpy/test/test_object_arrays.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/test/test_object_arrays.py @@ -0,0 +1,162 @@ +from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest + + +class AppTestObjectDtypes(BaseNumpyAppTest): + def test_scalar_from_object(self): + from numpy import array + import sys + class Polynomial(object): + def whatami(self): + return 'an object' + a = array(Polynomial()) + assert a.shape == () + assert a.sum().whatami() == 'an object' + + def test_uninitialized_object_array_is_filled_by_None(self): + import numpy as np + + a = np.ndarray([5], dtype="O") + + assert a[0] == None + + def test_object_arrays_add(self): + import numpy as np + + a = np.array(["foo"], dtype=object) + b = np.array(["bar"], dtype=object) + raises(TypeError, np.add, a, 1) + res = a + b + assert res[0] == "foobar" + + def test_bool_func(self): + import numpy as np + a = np.array(["foo"], dtype=object) + b = a and complex(1, -1) + assert b == complex(1, -1) + b = np.array(complex(1, -1)) and a + assert (b == a).all() + c = np.array([1, 2, 3]) + assert (a[0] != c[0]) + assert (c[0] != a[0]) + assert (a[0] > c[0]) + assert (not a[0] < c[0]) + assert (c[0] < a[0]) + assert (not c[0] > a[0]) + + def test_logical_ufunc(self): + import numpy as np + import sys + + if '__pypy__' in sys.builtin_module_names: + skip('need to refactor use of raw_xxx_op in types to make this work') + a = np.array(["foo"], dtype=object) + b = np.array([1], dtype=object) + d = np.array([complex(1, 10)], dtype=object) + c = np.logical_and(a, 1) + assert c.dtype == np.dtype('object') + assert c == 1 + c = np.logical_and(b, complex(1, -1)) + assert c.dtype == np.dtype('object') + assert c == complex(1, -1) + c = np.logical_and(d, b) + assert c == 1 + c = b & 1 + assert c.dtype == np.dtype('object') + assert (c == 1).all() + c = np.array(1) & b + assert (c == b).all() + + def test_reduce(self): + import numpy as np + class O(object): + def whatami(self): + return 'an object' + fiveOs = [O()] * 5 + a = np.array(fiveOs, dtype=object) + print np.maximum + b = np.maximum.reduce(a) + assert b is not None + + def test_complex_op(self): + import numpy as np + import sys + a = np.array(['abc', 'def'], dtype=object) + b = np.array([1, 2, 3], dtype=object) + c = np.array([complex(1, 1), complex(1, -1)], dtype=object) + for arg in (a,b,c): + assert (arg == np.real(arg)).all() + assert (0 == np.imag(arg)).all() + if '__pypy__' in sys.builtin_module_names: + skip('not implemented yet') + raises(AttributeError, np.conj, a) + res = np.conj(b) + assert (res == b).all() + res = np.conj(c) + assert res[0] == c[1] and res[1] == c[0] + + def test_keep_object_alive(self): + # only translated does it really test the gc + import numpy as np + import gc + class O(object): + def whatami(self): + return 'an object' + fiveOs = [O()] * 5 + a = np.array(fiveOs, dtype=object) + del fiveOs + gc.collect() + assert a[2].whatami() == 'an object' + + def test_array_interface(self): + import numpy as np + import sys + class DummyArray(object): + def __init__(self, interface, base=None): + self.__array_interface__ = interface + self.base = base + a = np.array([(1, 2, 3)], dtype='u4,u4,u4') + b = np.array([(1, 2, 3), (4, 5, 6), (7, 8, 9)], dtype='u4,u4,u4') + interface = dict(a.__array_interface__) + interface['shape'] = tuple([3]) + interface['strides'] = tuple([0]) + if '__pypy__' in sys.builtin_module_names: + skip('not implemented yet') + c = np.array(DummyArray(interface, base=a)) + c.dtype = a.dtype + #print c + assert (c == np.array([(1, 2, 3), (1, 2, 3), (1, 2, 3)], dtype='u4,u4,u4') ).all() + + def test_for_object_scalar_creation(self): + import numpy as np + import sys + a = np.object_() + b = np.object_(3) + b2 = np.object_(3.0) + c = np.object_([4, 5]) + d = np.array([None])[0] + assert a is None + assert type(b) is int + assert type(b2) is float + assert type(c) is np.ndarray + assert c.dtype == object + assert type(d) is type(None) + if '__pypy__' in sys.builtin_module_names: + skip('not implemented yet') + e = np.object_([None, {}, []]) + assert e.dtype == object + + def test_mem_array_creation_invalid_specification(self): + # while not specifically testing object dtype, this + # test segfaulted during ObjectType.store due to + # missing gc hooks + import numpy as np + import sys + ytype = np.object_ + if '__pypy__' in sys.builtin_module_names: + ytype = str + dt = np.dtype([('x', int), ('y', ytype)]) + # Correct way + a = np.array([(1, 'object')], dt) + # Wrong way - should complain about writing buffer to object dtype + raises(ValueError, np.array, [1, 'object'], dt) + diff --git a/pypy/module/micronumpy/test/test_selection.py b/pypy/module/micronumpy/test/test_selection.py --- a/pypy/module/micronumpy/test/test_selection.py +++ b/pypy/module/micronumpy/test/test_selection.py @@ -12,14 +12,11 @@ exp = sorted(range(len(exp)), key=exp.__getitem__) c = a.copy() res = a.argsort() - assert (res == exp).all(), '%r\n%r\n%r' % (a,res,exp) + assert (res == exp).all(), 'Failed sortng %r\na=%r\nres=%r\nexp=%r' % (dtype,a,res,exp) assert (a == c).all() # not modified a = arange(100, dtype=dtype) assert (a.argsort() == a).all() - import sys - if '__pypy__' in sys.builtin_module_names: - raises(NotImplementedError, 'arange(10,dtype="float16").argsort()') def test_argsort_ndim(self): from numpy import array @@ -63,14 +60,13 @@ 'i2', complex]: a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype) exp = sorted(list(a)) - res = a.copy() - res.sort() - assert (res == exp).all(), '%r\n%r\n%r' % (a,res,exp) + a.sort() + assert (a == exp).all(), 'Failed sorting %r\n%r\n%r' % (dtype, a, exp) a = arange(100, dtype=dtype) c = a.copy() a.sort() - assert (a == c).all() + assert (a == c).all(), 'Failed sortng %r\na=%r\nc=%r' % (dtype,a,c) def test_sort_nonnative(self): from numpy import array diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py --- a/pypy/module/micronumpy/test/test_ufuncs.py +++ b/pypy/module/micronumpy/test/test_ufuncs.py @@ -397,11 +397,11 @@ for i in range(3): assert min_c_b[i] == min(b[i], c) - def test_scalar(self): + def test_all_available(self): # tests that by calling all available ufuncs on scalars, none will # raise uncaught interp-level exceptions, (and crash the test) # and those that are uncallable can be accounted for. - # test on the four base-class dtypes: int, bool, float, complex + # test on the base-class dtypes: int, bool, float, complex, object # We need this test since they have no common base class. import numpy as np def find_uncallable_ufuncs(dtype): @@ -412,6 +412,11 @@ if isinstance(u, np.ufunc): try: u(* [array] * u.nin) + except AttributeError: + pass + except NotImplementedError: + print s + uncallable.add(s) except TypeError: assert s not in uncallable uncallable.add(s) @@ -427,6 +432,9 @@ 'fabs', 'fmod', 'invert', 'mod', 'logaddexp', 'logaddexp2', 'left_shift', 'right_shift', 'copysign', 'signbit', 'ceil', 'floor', 'trunc']) + assert find_uncallable_ufuncs('object') == set( + ['isnan', 'logaddexp2', 'copysign', 'isfinite', 'signbit', + 'isinf', 'logaddexp']) def test_int_only(self): from numpy import bitwise_and, array diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -9,6 +9,7 @@ from pypy.module.micronumpy.compile import FakeSpace, Parser, InterpreterState from pypy.module.micronumpy.base import W_NDimArray +py.test.skip('move these to pypyjit/test_pypy_c/test_micronumpy') class TestNumpyJit(LLJitMixin): graph = None diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -3,8 +3,9 @@ from pypy.interpreter.error import OperationError, oefmt from pypy.objspace.std.floatobject import float2string from pypy.objspace.std.complexobject import str_format +from pypy.interpreter.baseobjspace import W_Root, ObjSpace from rpython.rlib import clibffi, jit, rfloat, rcomplex -from rpython.rlib.objectmodel import specialize +from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rlib.rarithmetic import widen, byteswap, r_ulonglong, \ most_neg_value_of, LONG_BIT from rpython.rlib.rawstorage import (alloc_raw_storage, @@ -14,10 +15,12 @@ pack_float80, unpack_float80) from rpython.rlib.rstruct.nativefmttable import native_is_bigendian from rpython.rlib.rstruct.runpack import runpack -from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rtyper.annlowlevel import cast_instance_to_gcref,\ + cast_gcref_to_instance +from rpython.rtyper.lltypesystem import lltype, rffi, llmemory from rpython.tool.sourcetools import func_with_new_name from pypy.module.micronumpy import boxes -from pypy.module.micronumpy.concrete import SliceArray, VoidBoxStorage +from pypy.module.micronumpy.concrete import SliceArray, VoidBoxStorage, V_OBJECTSTORE from pypy.module.micronumpy.strides import calc_strides degToRad = math.pi / 180.0 @@ -109,10 +112,12 @@ return dispatcher class BaseType(object): - _immutable_fields_ = ['native'] + _immutable_fields_ = ['native', 'space'] - def __init__(self, native=True): + def __init__(self, space, native=True): + assert isinstance(space, ObjSpace) self.native = native + self.space = space def __repr__(self): return self.__class__.__name__ @@ -191,7 +196,7 @@ with arr as storage: self._write(storage, i, offset, self.unbox(box)) - def fill(self, storage, width, box, start, stop, offset): + def fill(self, storage, width, box, start, stop, offset, gcstruct): value = self.unbox(box) for i in xrange(start, stop, width): self._write(storage, i, offset, value) @@ -306,7 +311,7 @@ @raw_unary_op def rint(self, v): - float64 = Float64() + float64 = Float64(self.space) return float64.rint(float64.box(v)) class Bool(BaseType, Primitive): @@ -399,7 +404,7 @@ def round(self, v, decimals=0): if decimals != 0: return v - return Float64().box(self.unbox(v)) + return Float64(self.space).box(self.unbox(v)) class Integer(Primitive): _mixin_ = True @@ -444,7 +449,7 @@ self.T is rffi.LONG or self.T is rffi.LONGLONG): if v2 == -1 and v1 == self.for_computation(most_neg_value_of(self.T)): return self.box(0) - return self.box(v1 // v2) + return self.box(v1 / v2) @simple_binary_op def mod(self, v1, v2): @@ -1152,7 +1157,7 @@ with arr as storage: self._write(storage, i, offset, self.unbox(box)) - def fill(self, storage, width, box, start, stop, offset): + def fill(self, storage, width, box, start, stop, offset, gcstruct): value = self.unbox(box) for i in xrange(start, stop, width): self._write(storage, i, offset, value) @@ -1253,25 +1258,25 @@ def ge(self, v1, v2): return self._lt(v2, v1) or self._eq(v2, v1) - def _bool(self, v): + def _cbool(self, v): return bool(v[0]) or bool(v[1]) @raw_binary_op def logical_and(self, v1, v2): - return self._bool(v1) and self._bool(v2) + return self._cbool(v1) and self._cbool(v2) @raw_binary_op def logical_or(self, v1, v2): - return self._bool(v1) or self._bool(v2) + return self._cbool(v1) or self._cbool(v2) @raw_unary_op def logical_not(self, v): - return not self._bool(v) + return not self._cbool(v) @raw_binary_op def logical_xor(self, v1, v2): - a = self._bool(v1) - b = self._bool(v2) + a = self._cbool(v1) + b = self._cbool(v2) return (not b and a) or (not a and b) def min(self, v1, v2): @@ -1629,6 +1634,283 @@ BoxType = boxes.W_ComplexLongBox ComponentBoxType = boxes.W_FloatLongBox +_all_objs_for_tests = [] # for tests + +class ObjectType(Primitive, BaseType): + T = lltype.Signed + BoxType = boxes.W_ObjectBox + + def get_element_size(self): + return rffi.sizeof(lltype.Signed) + + def coerce(self, space, dtype, w_item): + if isinstance(w_item, boxes.W_ObjectBox): + return w_item + return boxes.W_ObjectBox(w_item) + + def coerce_subtype(self, space, w_subtype, w_item): + # return the item itself + return self.unbox(self.box(w_item)) + + def store(self, arr, i, offset, box): + if arr.gcstruct is V_OBJECTSTORE: + raise oefmt(self.space.w_NotImplementedError, + "cannot store object in array with no gc hook") + self._write(arr.storage, i, offset, self.unbox(box), + arr.gcstruct) + + def read(self, arr, i, offset, dtype=None): + return self.box(self._read(arr.storage, i, offset)) + + def byteswap(self, w_v): + return w_v + + @jit.dont_look_inside + def _write(self, storage, i, offset, w_obj, gcstruct): + # no GC anywhere in this function! + if we_are_translated(): + from rpython.rlib import rgc + rgc.ll_writebarrier(gcstruct) + value = rffi.cast(lltype.Signed, cast_instance_to_gcref(w_obj)) + else: + value = len(_all_objs_for_tests) + _all_objs_for_tests.append(w_obj) + raw_storage_setitem_unaligned(storage, i + offset, value) + + @jit.dont_look_inside + def _read(self, storage, i, offset): + res = raw_storage_getitem_unaligned(self.T, storage, i + offset) + if we_are_translated(): + gcref = rffi.cast(llmemory.GCREF, res) + w_obj = cast_gcref_to_instance(W_Root, gcref) + else: + w_obj = _all_objs_for_tests[res] + return w_obj + + def fill(self, storage, width, box, start, stop, offset, gcstruct): + value = self.unbox(box) + for i in xrange(start, stop, width): + self._write(storage, i, offset, value, gcstruct) + + def unbox(self, box): + if isinstance(box, self.BoxType): + return box.w_obj + else: + raise oefmt(self.space.w_NotImplementedError, + "object dtype cannot unbox %s", str(box)) + + @specialize.argtype(1) + def box(self, w_obj): + if isinstance(w_obj, W_Root): + pass + elif isinstance(w_obj, bool): + w_obj = self.space.newbool(w_obj) + elif isinstance(w_obj, int): + w_obj = self.space.newint(w_obj) + elif isinstance(w_obj, lltype.Number): + w_obj = self.space.newint(w_obj) + elif isinstance(w_obj, float): + w_obj = self.space.newfloat(w_obj) + elif w_obj is None: + w_obj = self.space.w_None + else: + raise oefmt(self.space.w_NotImplementedError, + "cannot create object array/scalar from lltype") + return self.BoxType(w_obj) + + @specialize.argtype(1, 2) + def box_complex(self, real, imag): + if isinstance(real, rffi.r_singlefloat): + real = rffi.cast(rffi.DOUBLE, real) + if isinstance(imag, rffi.r_singlefloat): + imag = rffi.cast(rffi.DOUBLE, imag) + w_obj = self.space.newcomplex(real, imag) + return self.BoxType(w_obj) + + def str_format(self, box): + return self.space.str_w(self.space.repr(self.unbox(box))) + + def runpack_str(self, space, s): + raise oefmt(space.w_NotImplementedError, + "fromstring not implemented for object type") + + def to_builtin_type(self, space, box): + assert isinstance(box, self.BoxType) + return box.w_obj + + @staticmethod + def for_computation(v): + return v + + @raw_binary_op + def eq(self, v1, v2): + return self.space.eq_w(v1, v2) + + @simple_binary_op + def max(self, v1, v2): + if self.space.is_true(self.space.ge(v1, v2)): + return v1 + return v2 + + @simple_binary_op + def min(self, v1, v2): + if self.space.is_true(self.space.le(v1, v2)): + return v1 + return v2 + + @raw_unary_op + def bool(self,v): + return self._obool(v) + + def _obool(self, v): + if self.space.is_true(v): + return True + return False + + @raw_binary_op + def logical_and(self, v1, v2): + if self._obool(v1): + return self.space.bool_w(v2) + return self.space.bool_w(v1) + + @raw_binary_op + def logical_or(self, v1, v2): + if self._obool(v1): + return self.space.bool_w(v1) + return self.space.bool_w(v2) + + @raw_unary_op + def logical_not(self, v): + return not self._obool(v) + + @raw_binary_op + def logical_xor(self, v1, v2): + a = self._obool(v1) + b = self._obool(v2) + return (not b and a) or (not a and b) + + @simple_binary_op + def bitwise_and(self, v1, v2): + return self.space.and_(v1, v2) + + @simple_binary_op + def bitwise_or(self, v1, v2): + return self.space.or_(v1, v2) + + @simple_binary_op + def bitwise_xor(self, v1, v2): + return self.space.xor(v1, v2) + + @simple_binary_op + def pow(self, v1, v2): + return self.space.pow(v1, v2, self.space.wrap(1)) + + @simple_unary_op + def reciprocal(self, v1): + return self.space.div(self.space.wrap(1.0), v1) + + @simple_unary_op + def sign(self, v): + zero = self.space.wrap(0) + one = self.space.wrap(1) + m_one = self.space.wrap(-1) + if self.space.is_true(self.space.gt(v, zero)): + return one + elif self.space.is_true(self.space.lt(v, zero)): + return m_one + else: + return zero + + @simple_unary_op + def real(self, v): + return v + + @simple_unary_op + def imag(self, v): + return 0 + + @simple_unary_op + def square(self, v): + return self.space.mul(v, v) + + @raw_binary_op + def le(self, v1, v2): + return self.space.bool_w(self.space.le(v1, v2)) + + @raw_binary_op + def ge(self, v1, v2): + return self.space.bool_w(self.space.ge(v1, v2)) + + @raw_binary_op + def lt(self, v1, v2): + return self.space.bool_w(self.space.lt(v1, v2)) + + @raw_binary_op + def gt(self, v1, v2): + return self.space.bool_w(self.space.gt(v1, v2)) + + @raw_binary_op + def ne(self, v1, v2): + return self.space.bool_w(self.space.ne(v1, v2)) + +def add_attributeerr_op(cls, op): + def func(self, *args): + raise oefmt(self.space.w_AttributeError, + "%s", op) + func.__name__ = 'object_' + op + setattr(cls, op, func) + +def add_unsupported_op(cls, op): + def func(self, *args): + raise oefmt(self.space.w_TypeError, + "ufunc '%s' not supported for input types", op) + func.__name__ = 'object_' + op + setattr(cls, op, func) + +def add_unary_op(cls, op, method): + @simple_unary_op + def func(self, w_v): + space = self.space + w_impl = space.lookup(w_v, method) + if w_impl is None: + raise oefmt(space.w_AttributeError, 'unknown op "%s" on object' % op) + return space.get_and_call_function(w_impl, w_v) + func.__name__ = 'object_' + op + setattr(cls, op, func) + +def add_space_unary_op(cls, op): + @simple_unary_op + def func(self, v): + return getattr(self.space, op)(v) + func.__name__ = 'object_' + op + setattr(cls, op, func) + +def add_space_binary_op(cls, op): + @simple_binary_op + def func(self, v1, v2): + return getattr(self.space, op)(v1, v2) + func.__name__ = 'object_' + op + setattr(cls, op, func) + +for op in ('copysign', 'isfinite', 'isinf', 'isnan', 'logaddexp', 'logaddexp2', + 'signbit'): + add_unsupported_op(ObjectType, op) +for op in ('arctan2', 'arccos', 'arccosh', 'arcsin', 'arcsinh', 'arctan', + 'arctanh', 'ceil', 'floor', 'cos', 'sin', 'tan', 'cosh', 'sinh', + 'tanh', 'radians', 'degrees', 'exp','exp2', 'expm1', 'fabs', + 'log', 'log10', 'log1p', 'log2', 'sqrt', 'trunc'): + add_attributeerr_op(ObjectType, op) +for op in ('abs', 'neg', 'pos', 'invert'): + add_space_unary_op(ObjectType, op) +for op, method in (('conj', 'descr_conjugate'), ('rint', 'descr_rint')): + add_unary_op(ObjectType, op, method) +for op in ('add', 'floordiv', 'div', 'mod', 'mul', 'sub', 'lshift', 'rshift'): + add_space_binary_op(ObjectType, op) + +ObjectType.fmax = ObjectType.max +ObjectType.fmin = ObjectType.min +ObjectType.fmod = ObjectType.mod + class FlexibleType(BaseType): def get_element_size(self): return rffi.sizeof(self.T) @@ -1758,7 +2040,7 @@ def bool(self, v): return bool(self.to_str(v)) - def fill(self, storage, width, box, start, stop, offset): + def fill(self, storage, width, box, start, stop, offset, gcstruct): for i in xrange(start, stop, width): self._store(storage, i, offset, box, width) @@ -1775,6 +2057,57 @@ raise OperationError(space.w_NotImplementedError, space.wrap( "coerce (probably from set_item) not implemented for unicode type")) + def store(self, arr, i, offset, box): + assert isinstance(box, boxes.W_UnicodeBox) + raise oefmt(self.space.w_NotImplementedError, "unicode type not completed") + + def read(self, arr, i, offset, dtype=None): + raise oefmt(self.space.w_NotImplementedError, "unicode type not completed") + + def str_format(self, item): + raise oefmt(self.space.w_NotImplementedError, "unicode type not completed") + + def to_builtin_type(self, space, box): + raise oefmt(self.space.w_NotImplementedError, "unicode type not completed") + + def eq(self, v1, v2): + raise oefmt(self.space.w_NotImplementedError, "unicode type not completed") + + def ne(self, v1, v2): + raise oefmt(self.space.w_NotImplementedError, "unicode type not completed") + + def lt(self, v1, v2): + raise oefmt(self.space.w_NotImplementedError, "unicode type not completed") + + def le(self, v1, v2): + raise oefmt(self.space.w_NotImplementedError, "unicode type not completed") + + def gt(self, v1, v2): + raise oefmt(self.space.w_NotImplementedError, "unicode type not completed") + + def ge(self, v1, v2): + raise oefmt(self.space.w_NotImplementedError, "unicode type not completed") + + def logical_and(self, v1, v2): + raise oefmt(self.space.w_NotImplementedError, "unicode type not completed") + + def logical_or(self, v1, v2): + raise oefmt(self.space.w_NotImplementedError, "unicode type not completed") + + def logical_not(self, v): + raise oefmt(self.space.w_NotImplementedError, "unicode type not completed") + + @str_binary_op + def logical_xor(self, v1, v2): + raise oefmt(self.space.w_NotImplementedError, "unicode type not completed") + + def bool(self, v): + raise oefmt(self.space.w_NotImplementedError, "unicode type not completed") + + def fill(self, storage, width, box, start, stop, offset, gcstruct): + raise oefmt(self.space.w_NotImplementedError, "unicode type not completed") + + class VoidType(FlexibleType): T = lltype.Char @@ -1882,6 +2215,9 @@ items_w = space.fixedview(w_item) elif isinstance(w_item, W_NDimArray) and w_item.is_scalar(): items_w = space.fixedview(w_item.get_scalar_value()) + elif space.isinstance_w(w_item, space.w_list): + raise oefmt(space.w_TypeError, + "expected a readable buffer object") else: # XXX support initializing from readable buffers items_w = [w_item] * len(dtype.fields) @@ -1913,7 +2249,7 @@ for k in range(size): storage[k + i + ofs] = box_storage[k + box.ofs] - def fill(self, storage, width, box, start, stop, offset): + def fill(self, storage, width, box, start, stop, offset, gcstruct): assert isinstance(box, boxes.W_VoidBox) assert width == box.dtype.elsize for i in xrange(start, stop, width): diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py --- a/pypy/module/micronumpy/ufuncs.py +++ b/pypy/module/micronumpy/ufuncs.py @@ -349,7 +349,7 @@ if dtype.is_flexible(): raise OperationError(space.w_TypeError, space.wrap('Not implemented for this type')) - if (self.int_only and not dtype.is_int() or + if (self.int_only and not (dtype.is_int() or dtype.is_object()) or not self.allow_bool and dtype.is_bool() or not self.allow_complex and dtype.is_complex()): raise oefmt(space.w_TypeError, @@ -378,6 +378,8 @@ w_val = self.func(calc_dtype, w_obj.get_scalar_value().convert_to(space, calc_dtype)) if out is None: + if res_dtype.is_object(): + w_val = w_obj.get_scalar_value() return w_val w_val = res_dtype.coerce(space, w_val) if out.is_scalar(): @@ -434,11 +436,20 @@ w_rhs = numpify(space, w_rhs) w_ldtype = _get_dtype(space, w_lhs) w_rdtype = _get_dtype(space, w_rhs) - if w_ldtype.is_str() and w_rdtype.is_str() and \ + if w_ldtype.is_object() or w_rdtype.is_object(): + pass + elif w_ldtype.is_str() and w_rdtype.is_str() and \ self.comparison_func: pass - elif (w_ldtype.is_str() or w_rdtype.is_str()) and \ + elif (w_ldtype.is_str()) and \ self.comparison_func and w_out is None: + if self.name in ('equal', 'less_equal', 'less'): + return space.wrap(False) + return space.wrap(True) + elif (w_rdtype.is_str()) and \ + self.comparison_func and w_out is None: + if self.name in ('not_equal','less', 'less_equal'): + return space.wrap(True) return space.wrap(False) elif w_ldtype.is_flexible() or w_rdtype.is_flexible(): if self.comparison_func: @@ -463,9 +474,9 @@ w_ldtype, w_rdtype, promote_to_float=self.promote_to_float, promote_bools=self.promote_bools) - if (self.int_only and (not w_ldtype.is_int() or - not w_rdtype.is_int() or - not calc_dtype.is_int()) or + if (self.int_only and (not (w_ldtype.is_int() or w_ldtype.is_object()) or + not (w_rdtype.is_int() or w_rdtype.is_object()) or + not (calc_dtype.is_int() or calc_dtype.is_object())) or not self.allow_bool and (w_ldtype.is_bool() or w_rdtype.is_bool()) or not self.allow_complex and (w_ldtype.is_complex() or @@ -643,7 +654,7 @@ # from frompyfunc pass # mimic NpyIter_AdvancedNew with a nditer - w_itershape = space.newlist([space.wrap(i) for i in iter_shape]) + w_itershape = space.newlist([space.wrap(i) for i in iter_shape]) nd_it = W_NDIter(space, space.newlist(inargs + outargs), w_flags, w_op_flags, w_op_dtypes, w_casting, w_op_axes, w_itershape) @@ -749,7 +760,7 @@ else: raise oefmt(space.w_TypeError, "a type-string for %s " \ "requires 1 typecode or %d typecode(s) before and %d" \ - " after the -> sign, not '%s'", self.name, self.nin, + " after the -> sign, not '%s'", self.name, self.nin, self.nout, type_tup) except KeyError: raise oefmt(space.w_ValueError, "unknown typecode in" \ @@ -773,11 +784,11 @@ for j in range(self.nargs): if dtypes[j] is not None and dtypes[j] != _dtypes[i+j]: allok = False - if allok: + if allok: break else: if len(self.funcs) > 1: - + dtypesstr = '' for d in dtypes: if d is None: @@ -787,7 +798,7 @@ _dtypesstr = ','.join(['%s%s%s' % (d.byteorder, d.kind, d.elsize) \ for d in _dtypes]) raise oefmt(space.w_TypeError, - "input dtype [%s] did not match any known dtypes [%s] ", + "input dtype [%s] did not match any known dtypes [%s] ", dtypesstr,_dtypesstr) i = 0 # Fill in empty dtypes @@ -807,7 +818,7 @@ assert isinstance(curarg, W_NDimArray) if len(arg_shapes[i]) != curarg.ndims(): # reshape - + sz = product(curarg.get_shape()) * curarg.get_dtype().elsize with curarg.implementation as storage: inargs[i] = W_NDimArray.from_shape_and_storage( @@ -865,7 +876,7 @@ "%s of gufunc was not specified", self.name, name, _i, core_dim_index, self.signature) target_dims.append(v) - arg_shapes.append(iter_shape + target_dims) + arg_shapes.append(iter_shape + target_dims) continue n = len(curarg.get_shape()) - num_dims if n < 0: @@ -907,7 +918,7 @@ raise oefmt(space.w_ValueError, "%s: %s operand %d has a " "mismatch in its core dimension %d, with gufunc " "signature %s (expected %d, got %d)", - self.name, name, _i, j, + self.name, name, _i, j, self.signature, matched_dims[core_dim_index], dims_to_match[core_dim_index]) #print 'adding',iter_shape,'+',dims_to_match,'to arg_shapes' @@ -950,6 +961,10 @@ return dt1 if dt1 is None: return dt2 + + if dt1.num == NPY.OBJECT or dt2.num == NPY.OBJECT: + return get_dtype_cache(space).w_objectdtype + # dt1.num should be <= dt2.num if dt1.num > dt2.num: dt1, dt2 = dt2, dt1 @@ -1032,6 +1047,8 @@ @jit.unroll_safe def find_unaryop_result_dtype(space, dt, promote_to_float=False, promote_bools=False, promote_to_largest=False): + if dt.is_object(): + return dt if promote_to_largest: if dt.kind == NPY.GENBOOLLTR or dt.kind == NPY.SIGNEDLTR: if dt.elsize * 8 < LONG_BIT: @@ -1064,6 +1081,7 @@ uint64_dtype = get_dtype_cache(space).w_uint64dtype complex_dtype = get_dtype_cache(space).w_complex128dtype float_dtype = get_dtype_cache(space).w_float64dtype + object_dtype = get_dtype_cache(space).w_objectdtype if isinstance(w_obj, boxes.W_GenericBox): dtype = w_obj.get_dtype(space) return find_binop_result_dtype(space, dtype, current_guess) @@ -1097,9 +1115,10 @@ return variable_dtype(space, 'S%d' % space.len_w(w_obj)) return current_guess - raise oefmt(space.w_NotImplementedError, - 'unable to create dtype from objects, "%T" instance not ' - 'supported', w_obj) + return object_dtype + #raise oefmt(space.w_NotImplementedError, + # 'unable to create dtype from objects, "%T" instance not ' + # 'supported', w_obj) def ufunc_dtype_caller(space, ufunc_name, op_name, nin, comparison_func, @@ -1263,7 +1282,7 @@ w_identity=None, name='', doc='', stack_inputs=False): ''' frompyfunc(func, nin, nout) #cpython numpy compatible frompyfunc(func, nin, nout, dtypes=None, signature='', - identity=None, name='', doc='', + identity=None, name='', doc='', stack_inputs=False) Takes an arbitrary Python function and returns a ufunc. @@ -1282,7 +1301,7 @@ dtypes: None or [dtype, ...] of the input, output args for each function, or 'match' to force output to exactly match input dtype Note that 'match' is a pypy-only extension to allow non-object - return dtypes + return dtypes signature*: str, default='' The mapping of input args to output args, defining the inner-loop indexing. If it is empty, the func operates on scalars @@ -1293,7 +1312,7 @@ stack_inputs*: boolean, whether the function is of the form out = func(*in) False or - func(*[in + out]) True + func(*[in + out]) True only one of out_dtype or signature may be specified diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -686,6 +686,15 @@ lambda_customtrace = lambda: customtrace """ + at specialize.ll() +def ll_writebarrier(gc_obj): + """Use together with custom tracers. When you update some object pointer + stored in raw memory, you must call this function on 'gc_obj', which must + be the object of type TP with the custom tracer (*not* the value stored!). + This makes sure that the custom hook will be called again.""" + from rpython.rtyper.lltypesystem.lloperation import llop + llop.gc_writebarrier(lltype.Void, gc_obj) + class RegisterGcTraceEntry(ExtRegistryEntry): _about_ = register_custom_trace_hook From noreply at buildbot.pypy.org Thu Apr 23 21:25:10 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 23 Apr 2015 21:25:10 +0200 (CEST) Subject: [pypy-commit] pypy default: skip one more object non-implemented feature Message-ID: <20150423192510.8982D1C1229@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76906:86b0eb2de56f Date: 2015-04-23 22:24 +0300 http://bitbucket.org/pypy/pypy/changeset/86b0eb2de56f/ Log: skip one more object non-implemented feature diff --git a/pypy/module/micronumpy/test/test_selection.py b/pypy/module/micronumpy/test/test_selection.py --- a/pypy/module/micronumpy/test/test_selection.py +++ b/pypy/module/micronumpy/test/test_selection.py @@ -218,6 +218,7 @@ def test_sort_objects(self): # test object array sorts. + skip('object type not supported yet') from numpy import empty try: a = empty((101,), dtype=object) From noreply at buildbot.pypy.org Thu Apr 23 21:25:11 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 23 Apr 2015 21:25:11 +0200 (CEST) Subject: [pypy-commit] pypy default: fix astype creation to match memory layout of source array Message-ID: <20150423192511.C17731C1229@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76907:d198d926afb8 Date: 2015-04-23 22:24 +0300 http://bitbucket.org/pypy/pypy/changeset/d198d926afb8/ Log: fix astype creation to match memory layout of source array diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py --- a/pypy/module/micronumpy/concrete.py +++ b/pypy/module/micronumpy/concrete.py @@ -328,8 +328,11 @@ return ArrayBuffer(self, readonly) def astype(self, space, dtype): - strides, backstrides = calc_strides(self.get_shape(), dtype, - self.order) + # we want to create a new array, but must respect the strides + # in self. So find a factor of the itemtype.elsize, and use this + factor = float(dtype.elsize) / self.dtype.elsize + strides = [int(factor*s) for s in self.get_strides()] + backstrides = [int(factor*s) for s in self.get_backstrides()] impl = ConcreteArray(self.get_shape(), dtype, self.order, strides, backstrides) loop.setslice(space, impl.get_shape(), impl, self) diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -2183,7 +2183,8 @@ assert b.dtype == 'bool' a = arange(6, dtype='f4').reshape(2,3) - b = a.astype('i4') + b = a.T.astype('i4') + assert (a.T.strides == b.strides) a = array('x').astype('S3').dtype assert a.itemsize == 3 From noreply at buildbot.pypy.org Thu Apr 23 22:35:51 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 23 Apr 2015 22:35:51 +0200 (CEST) Subject: [pypy-commit] pypy.org extradoc: update the values Message-ID: <20150423203551.A9CFC1C12C8@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r595:fe9e99ca741b Date: 2015-04-23 22:36 +0200 http://bitbucket.org/pypy/pypy.org/changeset/fe9e99ca741b/ Log: update the values diff --git a/don4.html b/don4.html --- a/don4.html +++ b/don4.html @@ -9,7 +9,7 @@ @@ -17,7 +17,7 @@ 2nd call: - $22674 of $80000 (28.3%) + $28642 of $80000 (35.8%)

From noreply at buildbot.pypy.org Thu Apr 23 23:26:55 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Thu, 23 Apr 2015 23:26:55 +0200 (CEST) Subject: [pypy-commit] pypy py3k: Fix the _curses module to work with Python3. Message-ID: <20150423212655.2DB7F1C0EB2@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r76908:105de1f4b6b3 Date: 2015-04-23 23:24 +0200 http://bitbucket.org/pypy/pypy/changeset/105de1f4b6b3/ Log: Fix the _curses module to work with Python3. Also fix test_curses which is still in CPython3.2... Apply fix from CPython issue20358. diff --git a/lib-python/3/test/test_curses.py b/lib-python/3/test/test_curses.py --- a/lib-python/3/test/test_curses.py +++ b/lib-python/3/test/test_curses.py @@ -115,8 +115,8 @@ stdscr.notimeout(1) win2.overlay(win) win2.overwrite(win) - win2.overlay(win, 1, 2, 3, 3, 2, 1) - win2.overwrite(win, 1, 2, 3, 3, 2, 1) + win2.overlay(win, 1, 2, 2, 1, 3, 3) + win2.overwrite(win, 1, 2, 2, 1, 3, 3) stdscr.redrawln(1,2) stdscr.scrollok(1) diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py --- a/lib_pypy/_curses.py +++ b/lib_pypy/_curses.py @@ -484,13 +484,13 @@ def _chtype(ch): return int(ffi.cast("chtype", ch)) -def _texttype(text): - if isinstance(text, str): +def _bytestype(text): + if isinstance(text, bytes): return text - elif isinstance(text, unicode): - return str(text) # default encoding + elif isinstance(text, str): + return text.encode() else: - raise TypeError("str or unicode expected, got a '%s' object" + raise TypeError("bytes or str expected, got a '%s' object" % (type(text).__name__,)) @@ -606,7 +606,7 @@ @_argspec(1, 1, 2) def addstr(self, y, x, text, attr=None): - text = _texttype(text) + text = _bytestype(text) if attr is not None: attr_old = lib.getattrs(self._win) lib.wattrset(self._win, attr) @@ -620,7 +620,7 @@ @_argspec(2, 1, 2) def addnstr(self, y, x, text, n, attr=None): - text = _texttype(text) + text = _bytestype(text) if attr is not None: attr_old = lib.getattrs(self._win) lib.wattrset(self._win, attr) @@ -799,7 +799,7 @@ @_argspec(1, 1, 2) def insstr(self, y, x, text, attr=None): - text = _texttype(text) + text = _bytestype(text) if attr is not None: attr_old = lib.getattrs(self._win) lib.wattrset(self._win, attr) @@ -813,7 +813,7 @@ @_argspec(2, 1, 2) def insnstr(self, y, x, text, n, attr=None): - text = _texttype(text) + text = _bytestype(text) if attr is not None: attr_old = lib.getattrs(self._win) lib.wattrset(self._win, attr) @@ -1221,7 +1221,7 @@ def putp(text): - text = _texttype(text) + text = _bytestype(text) return _check_ERR(lib.putp(text), "putp") @@ -1347,17 +1347,17 @@ def tigetflag(capname): _ensure_initialised_setupterm() - return lib.tigetflag(capname) + return lib.tigetflag(capname.encode()) def tigetnum(capname): _ensure_initialised_setupterm() - return lib.tigetnum(capname) + return lib.tigetnum(capname.encode()) def tigetstr(capname): _ensure_initialised_setupterm() - val = lib.tigetstr(capname) + val = lib.tigetstr(capname.encode()) if int(ffi.cast("intptr_t", val)) in (0, -1): return None return ffi.string(val) From noreply at buildbot.pypy.org Fri Apr 24 01:39:48 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Fri, 24 Apr 2015 01:39:48 +0200 (CEST) Subject: [pypy-commit] pypy can_cast: extract as_dtype() from result_type() so it can be used in can_cast() as well Message-ID: <20150423233948.4227C1C1229@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: can_cast Changeset: r76909:6048923edd3e Date: 2015-04-23 14:37 +0100 http://bitbucket.org/pypy/pypy/changeset/6048923edd3e/ Log: extract as_dtype() from result_type() so it can be used in can_cast() as well diff --git a/pypy/module/micronumpy/arrayops.py b/pypy/module/micronumpy/arrayops.py --- a/pypy/module/micronumpy/arrayops.py +++ b/pypy/module/micronumpy/arrayops.py @@ -296,21 +296,25 @@ raise oefmt(space.w_ValueError, "at least one array or dtype is required") result = None for w_arg in args_w: - if isinstance(w_arg, W_NDimArray): - dtype = w_arg.get_dtype() - elif isinstance(w_arg, W_GenericBox) or ( - space.isinstance_w(w_arg, space.w_int) or - space.isinstance_w(w_arg, space.w_float) or - space.isinstance_w(w_arg, space.w_complex) or - space.isinstance_w(w_arg, space.w_long) or - space.isinstance_w(w_arg, space.w_bool)): - dtype = ufuncs.find_dtype_for_scalar(space, w_arg) - else: - dtype = space.interp_w(descriptor.W_Dtype, - space.call_function(space.gettypefor(descriptor.W_Dtype), w_arg)) + dtype = as_dtype(space, w_arg) result = ufuncs.find_binop_result_dtype(space, result, dtype) return result @unwrap_spec(casting=str) def can_cast(space, w_from, w_totype, casting='safe'): return space.w_True + +def as_dtype(space, w_arg): + # roughly equivalent to CNumPy's PyArray_DescrConverter2 + if isinstance(w_arg, W_NDimArray): + return w_arg.get_dtype() + elif isinstance(w_arg, W_GenericBox) or ( + space.isinstance_w(w_arg, space.w_int) or + space.isinstance_w(w_arg, space.w_float) or + space.isinstance_w(w_arg, space.w_complex) or + space.isinstance_w(w_arg, space.w_long) or + space.isinstance_w(w_arg, space.w_bool)): + return ufuncs.find_dtype_for_scalar(space, w_arg) + else: + return space.interp_w(descriptor.W_Dtype, + space.call_function(space.gettypefor(descriptor.W_Dtype), w_arg)) diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -4049,4 +4049,3 @@ raises(TypeError, np.can_cast, 'i4', None) raises(TypeError, np.can_cast, None, 'i4') - From noreply at buildbot.pypy.org Fri Apr 24 01:39:49 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Fri, 24 Apr 2015 01:39:49 +0200 (CEST) Subject: [pypy-commit] pypy can_cast: create W_Dtype.can_cast_to() Message-ID: <20150423233949.60F791C1229@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: can_cast Changeset: r76910:9e8922d9dce5 Date: 2015-04-23 20:32 +0100 http://bitbucket.org/pypy/pypy/changeset/9e8922d9dce5/ Log: create W_Dtype.can_cast_to() diff --git a/pypy/module/micronumpy/arrayops.py b/pypy/module/micronumpy/arrayops.py --- a/pypy/module/micronumpy/arrayops.py +++ b/pypy/module/micronumpy/arrayops.py @@ -302,7 +302,10 @@ @unwrap_spec(casting=str) def can_cast(space, w_from, w_totype, casting='safe'): - return space.w_True + target = as_dtype(space, w_totype) + origin = as_dtype(space, w_from) # XXX + return space.wrap(origin.can_cast_to(target)) + def as_dtype(space, w_arg): # roughly equivalent to CNumPy's PyArray_DescrConverter2 diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py --- a/pypy/module/micronumpy/descriptor.py +++ b/pypy/module/micronumpy/descriptor.py @@ -87,6 +87,9 @@ def box_complex(self, real, imag): return self.itemtype.box_complex(real, imag) + def can_cast_to(self, other): + return True + def coerce(self, space, w_item): return self.itemtype.coerce(space, self, w_item) From noreply at buildbot.pypy.org Fri Apr 24 01:39:51 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Fri, 24 Apr 2015 01:39:51 +0200 (CEST) Subject: [pypy-commit] pypy can_cast: hg merge default Message-ID: <20150423233951.368DB1C1229@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: can_cast Changeset: r76911:1894fda8b437 Date: 2015-04-23 21:57 +0100 http://bitbucket.org/pypy/pypy/changeset/1894fda8b437/ Log: hg merge default diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,12 +5,46 @@ .. this is a revision shortly after release-2.5.1 .. startrev: cb01edcb59414d9d93056e54ed060673d24e67c1 -Issue #2017: on non-Linux-x86 platforms, reduced the memory impact of +issue2005: +ignore errors on closing random file handles while importing a module (cpython compatibility) + +issue2013: +added constants to _ssl for TLS 1.1 and 1.2 + +issue2014: +Add PyLong_FromUnicode to cpyext. + +issue2017: +On non-Linux-x86 platforms, reduced the memory impact of creating a lot of greenlets/tasklets. Particularly useful on Win32 and on ARM, where you used to get a MemoryError after only 2500-5000 greenlets (the 32-bit address space is exhausted). +Update gdb_pypy for python3 (gdb comatability) + +Merged rstrategies into rpython which provides a library for Storage Strategies + +Support unicode strings in numpy.dtype creation i.e. np.dtype(u'int64') + +Various rpython cleanups for vmprof support + +issue2019: +Fix isspace as called by rpython unicode.strip() + +issue2023: +In the cpyext 'Concrete Object Layer' API, +don't call methods on the object (which can be overriden), +but directly on the concrete base type. + +issue2029: +Hide the default_factory attribute in a dict + +issue2027: +Better document pyinteractive and add --withmod-time + .. branch: gc-incminimark-pinning-improve + +branch gc-incminimark-pinning-improve: Object Pinning is now used in `bz2` and `rzlib` (therefore also affects Python's `zlib`). In case the data to compress/decompress is inside the nursery (incminimark) it no longer needs to create a non-moving copy of it. This saves @@ -18,8 +52,18 @@ is introduced (`PYPY_GC_MAX_PINNED`) primarily for debugging purposes. .. branch: refactor-pycall + +branch refactor-pycall: Make `*`-unpacking in RPython function calls completely equivalent to passing the tuple's elements as arguments. In other words, `f(*(a, b))` now behaves exactly like `f(a, b)`. .. branch: issue2018 +branch issue2018: +Allow prebuilt rpython dict with function values + +.. branch: vmprof +.. Merged but then backed out, hopefully it will return as vmprof2 + +.. branch: object-dtype2 +Extend numpy dtypes to allow using objects with associated garbage collection hook diff --git a/pypy/goal/targetnumpystandalone.py b/pypy/goal/targetnumpystandalone.py deleted file mode 100644 --- a/pypy/goal/targetnumpystandalone.py +++ /dev/null @@ -1,43 +0,0 @@ - -""" Usage: - -./targetnumpystandalone-c array_size - -Will execute a give numpy bytecode. Arrays will be ranges (in float) modulo 10, -constants would be consecutive starting from one. - -Bytecode should contain letters 'a' 'l' and 'f' so far and be correct -""" - -import time -from pypy.module.micronumpy.compile import numpy_compile -from rpython.jit.codewriter.policy import JitPolicy -from rpython.rtyper.annlowlevel import hlstr - -def entry_point(argv): - if len(argv) != 3: - print __doc__ - return 1 - try: - size = int(argv[2]) - except ValueError: - print "INVALID LITERAL FOR INT:", argv[2] - print __doc__ - return 3 - t0 = time.time() - main(argv[0], size) - print "bytecode:", argv[0], "size:", size - print "took:", time.time() - t0 - return 0 - -def main(bc, size): - if not isinstance(bc, str): - bc = hlstr(bc) # for tests - a = numpy_compile(bc, size) - a = a.compute() - -def target(*args): - return entry_point, None - -def jitpolicy(driver): - return JitPolicy() diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py --- a/pypy/module/micronumpy/__init__.py +++ b/pypy/module/micronumpy/__init__.py @@ -31,6 +31,9 @@ for c in ['MAXDIMS', 'CLIP', 'WRAP', 'RAISE']: interpleveldefs[c] = 'space.wrap(constants.%s)' % c + def startup(self, space): + from pypy.module.micronumpy.concrete import _setup + _setup() class UMathModule(MixedModule): appleveldefs = {} diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py --- a/pypy/module/micronumpy/base.py +++ b/pypy/module/micronumpy/base.py @@ -34,11 +34,13 @@ @staticmethod def from_shape(space, shape, dtype, order='C', w_instance=None, zero=True): - from pypy.module.micronumpy import concrete + from pypy.module.micronumpy import concrete, descriptor, boxes from pypy.module.micronumpy.strides import calc_strides strides, backstrides = calc_strides(shape, dtype.base, order) impl = concrete.ConcreteArray(shape, dtype.base, order, strides, backstrides, zero=zero) + if dtype == descriptor.get_dtype_cache(space).w_objectdtype: + impl.fill(space, boxes.W_ObjectBox(space.w_None)) if w_instance: return wrap_impl(space, space.type(w_instance), w_instance, impl) return W_NDimArray(impl) @@ -123,7 +125,7 @@ def get_shape(self): return self.implementation.get_shape() - def get_dtype(self): + def get_dtype(self, space=None): return self.implementation.dtype def get_order(self): diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py --- a/pypy/module/micronumpy/boxes.py +++ b/pypy/module/micronumpy/boxes.py @@ -607,6 +607,19 @@ # arr.storage[i] = arg[i] return W_UnicodeBox(arr, 0, arr.dtype) +class W_ObjectBox(W_GenericBox): + descr__new__, _get_dtype, descr_reduce = new_dtype_getter(NPY.OBJECT) + + def __init__(self, w_obj): + self.w_obj = w_obj + + def convert_to(self, space, dtype): + if dtype.is_bool(): + return W_BoolBox(space.bool_w(self.w_obj)) + return self # XXX + + def descr__getattr__(self, space, w_key): + return space.getattr(self.w_obj, w_key) W_GenericBox.typedef = TypeDef("numpy.generic", __new__ = interp2app(W_GenericBox.descr__new__.im_func), @@ -856,3 +869,9 @@ __new__ = interp2app(W_UnicodeBox.descr__new__unicode_box.im_func), __len__ = interp2app(W_UnicodeBox.descr_len), ) + +W_ObjectBox.typedef = TypeDef("numpy.object_", W_ObjectBox.typedef, + __new__ = interp2app(W_ObjectBox.descr__new__.im_func), + __getattr__ = interp2app(W_ObjectBox.descr__getattr__), +) + diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py --- a/pypy/module/micronumpy/compile.py +++ b/pypy/module/micronumpy/compile.py @@ -3,7 +3,7 @@ """ import re from pypy.interpreter import special -from pypy.interpreter.baseobjspace import InternalSpaceCache, W_Root +from pypy.interpreter.baseobjspace import InternalSpaceCache, W_Root, ObjSpace from pypy.interpreter.error import OperationError from rpython.rlib.objectmodel import specialize, instantiate from rpython.rlib.nonconst import NonConstant @@ -47,7 +47,7 @@ def lookup(self, name): return self.getdictvalue(self, name) -class FakeSpace(object): +class FakeSpace(ObjSpace): w_ValueError = W_TypeObject("ValueError") w_TypeError = W_TypeObject("TypeError") w_IndexError = W_TypeObject("IndexError") @@ -67,6 +67,7 @@ w_unicode = W_TypeObject("unicode") w_complex = W_TypeObject("complex") w_dict = W_TypeObject("dict") + w_object = W_TypeObject("object") def __init__(self): """NOT_RPYTHON""" @@ -88,7 +89,8 @@ return self.wrap(len(w_obj.items)) def getattr(self, w_obj, w_attr): - return StringObject(NonConstant('foo')) + assert isinstance(w_attr, StringObject) + return w_obj.getdictvalue(self, w_attr.v) def isinstance_w(self, w_obj, w_tp): try: diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py --- a/pypy/module/micronumpy/concrete.py +++ b/pypy/module/micronumpy/concrete.py @@ -1,11 +1,11 @@ from pypy.interpreter.error import OperationError, oefmt -from rpython.rlib import jit +from rpython.rlib import jit, rgc from rpython.rlib.buffer import Buffer -from rpython.rlib.debug import make_sure_not_resized +from rpython.rlib.debug import make_sure_not_resized, debug_print from rpython.rlib.rawstorage import alloc_raw_storage, free_raw_storage, \ raw_storage_getitem, raw_storage_setitem, RAW_STORAGE -from rpython.rtyper.lltypesystem import rffi, lltype -from pypy.module.micronumpy import support, loop +from rpython.rtyper.lltypesystem import rffi, lltype, llmemory +from pypy.module.micronumpy import support, loop, constants as NPY from pypy.module.micronumpy.base import convert_to_array, W_NDimArray, \ ArrayArgumentException from pypy.module.micronumpy.iterators import ArrayIter @@ -13,11 +13,13 @@ RecordChunk, calc_strides, calc_new_strides, shape_agreement, calculate_broadcast_strides, calc_backstrides) from rpython.rlib.objectmodel import keepalive_until_here +from rpython.rtyper.annlowlevel import cast_gcref_to_instance +from pypy.interpreter.baseobjspace import W_Root class BaseConcreteArray(object): _immutable_fields_ = ['dtype?', 'storage', 'start', 'size', 'shape[*]', - 'strides[*]', 'backstrides[*]', 'order'] + 'strides[*]', 'backstrides[*]', 'order', 'gcstruct'] start = 0 parent = None flags = 0 @@ -326,13 +328,54 @@ return ArrayBuffer(self, readonly) def astype(self, space, dtype): - strides, backstrides = calc_strides(self.get_shape(), dtype, - self.order) + # we want to create a new array, but must respect the strides + # in self. So find a factor of the itemtype.elsize, and use this + factor = float(dtype.elsize) / self.dtype.elsize + strides = [int(factor*s) for s in self.get_strides()] + backstrides = [int(factor*s) for s in self.get_backstrides()] impl = ConcreteArray(self.get_shape(), dtype, self.order, strides, backstrides) loop.setslice(space, impl.get_shape(), impl, self) return impl +OBJECTSTORE = lltype.GcStruct('ObjectStore', + ('length', lltype.Signed), + ('step', lltype.Signed), + ('storage', llmemory.Address), + rtti=True) +offset_of_storage = llmemory.offsetof(OBJECTSTORE, 'storage') +offset_of_length = llmemory.offsetof(OBJECTSTORE, 'length') +offset_of_step = llmemory.offsetof(OBJECTSTORE, 'step') + +V_OBJECTSTORE = lltype.nullptr(OBJECTSTORE) + +def customtrace(gc, obj, callback, arg): + #debug_print('in customtrace w/obj', obj) + length = (obj + offset_of_length).signed[0] + step = (obj + offset_of_step).signed[0] + storage = (obj + offset_of_storage).address[0] + #debug_print('tracing', length, 'objects in ndarray.storage') + i = 0 + while i < length: + gc._trace_callback(callback, arg, storage) + storage += step + i += 1 + +lambda_customtrace = lambda: customtrace + +def _setup(): + rgc.register_custom_trace_hook(OBJECTSTORE, lambda_customtrace) + + at jit.dont_look_inside +def _create_objectstore(storage, length, elsize): + gcstruct = lltype.malloc(OBJECTSTORE) + # JIT does not support cast_ptr_to_adr + gcstruct.storage = llmemory.cast_ptr_to_adr(storage) + #print 'create gcstruct',gcstruct,'with storage',storage,'as',gcstruct.storage + gcstruct.length = length + gcstruct.step = elsize + return gcstruct + class ConcreteArrayNotOwning(BaseConcreteArray): def __init__(self, shape, dtype, order, strides, backstrides, storage, start=0): @@ -347,10 +390,11 @@ self.backstrides = backstrides self.storage = storage self.start = start + self.gcstruct = V_OBJECTSTORE def fill(self, space, box): self.dtype.itemtype.fill(self.storage, self.dtype.elsize, - box, 0, self.size, 0) + box, 0, self.size, 0, self.gcstruct) def set_shape(self, space, orig_array, new_shape): strides, backstrides = calc_strides(new_shape, self.dtype, @@ -374,17 +418,24 @@ def base(self): return None - class ConcreteArray(ConcreteArrayNotOwning): def __init__(self, shape, dtype, order, strides, backstrides, storage=lltype.nullptr(RAW_STORAGE), zero=True): + gcstruct = V_OBJECTSTORE if storage == lltype.nullptr(RAW_STORAGE): - storage = dtype.itemtype.malloc(support.product(shape) * - dtype.elsize, zero=zero) + length = support.product(shape) + if dtype.num == NPY.OBJECT: + storage = dtype.itemtype.malloc(length * dtype.elsize, zero=True) + gcstruct = _create_objectstore(storage, length, dtype.elsize) + else: + storage = dtype.itemtype.malloc(length * dtype.elsize, zero=zero) ConcreteArrayNotOwning.__init__(self, shape, dtype, order, strides, backstrides, storage) + self.gcstruct = gcstruct def __del__(self): + if self.gcstruct: + self.gcstruct.length = 0 free_raw_storage(self.storage, track_allocation=False) @@ -423,6 +474,7 @@ parent = parent.parent # one level only self.parent = parent self.storage = parent.storage + self.gcstruct = parent.gcstruct self.order = parent.order self.dtype = dtype self.size = support.product(shape) * self.dtype.elsize @@ -480,6 +532,7 @@ class VoidBoxStorage(BaseConcreteArray): def __init__(self, size, dtype): self.storage = alloc_raw_storage(size) + self.gcstruct = V_OBJECTSTORE self.dtype = dtype self.size = size diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py --- a/pypy/module/micronumpy/ctors.py +++ b/pypy/module/micronumpy/ctors.py @@ -38,6 +38,34 @@ raise oefmt(space.w_ValueError, "object __array__ method not producing an array") +def try_interface_method(space, w_object): + try: + w_interface = space.getattr(w_object, space.wrap("__array_interface__")) + except OperationError, e: + if e.match(space, space.w_AttributeError): + return None + raise + if w_interface is None: + # happens from compile.py + return None + version = space.int_w(space.finditem(w_interface, space.wrap("version"))) + if version < 3: + raise oefmt(space.w_NotImplementedError, + "__array_interface__ version %d not supported", version) + # make a view into the data + w_shape = space.finditem(w_interface, space.wrap('shape')) + w_dtype = space.finditem(w_interface, space.wrap('typestr')) + w_descr = space.finditem(w_interface, space.wrap('descr')) + data_w = space.listview(space.finditem(w_interface, space.wrap('data'))) + w_strides = space.finditem(w_interface, space.wrap('strides')) + shape = [space.int_w(i) for i in space.listview(w_shape)] + dtype = descriptor.decode_w_dtype(space, w_dtype) + rw = space.is_true(data_w[1]) + #print 'create view from shape',shape,'dtype',dtype,'descr',w_descr,'data',data_w[0],'rw',rw + raise oefmt(space.w_NotImplementedError, + "creating array from __array_interface__ not supported yet") + return + @unwrap_spec(ndmin=int, copy=bool, subok=bool) def array(space, w_object, w_dtype=None, copy=True, w_order=None, subok=False, @@ -63,7 +91,11 @@ # continue with w_array, but do further operations in place w_object = w_array copy = False - + if not isinstance(w_object, W_NDimArray): + w_array = try_interface_method(space, w_object) + if w_array is not None: + w_object = w_array + copy = False dtype = descriptor.decode_w_dtype(space, w_dtype) if space.is_none(w_order): diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py --- a/pypy/module/micronumpy/descriptor.py +++ b/pypy/module/micronumpy/descriptor.py @@ -6,7 +6,7 @@ from pypy.interpreter.typedef import (TypeDef, GetSetProperty, interp_attrproperty, interp_attrproperty_w) from rpython.rlib import jit -from rpython.rlib.objectmodel import specialize, compute_hash +from rpython.rlib.objectmodel import specialize, compute_hash, we_are_translated from rpython.rlib.rarithmetic import r_longlong, r_ulonglong from pypy.module.micronumpy import types, boxes, base, support, constants as NPY from pypy.module.micronumpy.appbridge import get_appbridge_cache @@ -56,7 +56,7 @@ self.char = char self.w_box_type = w_box_type if byteorder is None: - if itemtype.get_element_size() == 1: + if itemtype.get_element_size() == 1 or isinstance(itemtype, types.ObjectType): byteorder = NPY.IGNORE else: byteorder = NPY.NATIVE @@ -115,6 +115,9 @@ def is_str(self): return self.num == NPY.STRING + def is_object(self): + return self.num == NPY.OBJECT + def is_str_or_unicode(self): return self.num == NPY.STRING or self.num == NPY.UNICODE @@ -431,7 +434,7 @@ self.names.append(name) self.fields[name] = offset, dtype - self.itemtype = types.RecordType() + self.itemtype = types.RecordType(space) if self.is_flexible(): self.elsize = size @@ -446,7 +449,7 @@ endian = NPY.OPPBYTE if self.is_native() else NPY.NATBYTE elif newendian != NPY.IGNORE: endian = newendian - itemtype = self.itemtype.__class__(endian in (NPY.NATIVE, NPY.NATBYTE)) + itemtype = self.itemtype.__class__(space, endian in (NPY.NATIVE, NPY.NATBYTE)) fields = self.fields if fields is None: fields = {} @@ -485,7 +488,7 @@ fields[fldname] = (offset, subdtype) offset += subdtype.elsize names.append(fldname) - return W_Dtype(types.RecordType(), NPY.VOID, NPY.VOIDLTR, NPY.VOIDLTR, + return W_Dtype(types.RecordType(space), NPY.VOID, NPY.VOIDLTR, NPY.VOIDLTR, space.gettypefor(boxes.W_VoidBox), names=names, fields=fields, elsize=offset) @@ -496,8 +499,17 @@ def dtype_from_spec(space, w_spec): - w_lst = get_appbridge_cache(space).call_method(space, - 'numpy.core._internal', '_commastring', Arguments(space, [w_spec])) + + if we_are_translated(): + w_lst = get_appbridge_cache(space).call_method(space, + 'numpy.core._internal', '_commastring', Arguments(space, [w_spec])) + else: + # testing, handle manually + if space.eq_w(w_spec, space.wrap('u4,u4,u4')): + w_lst = space.newlist([space.wrap('u4')]*3) + else: + raise oefmt(space.w_RuntimeError, + "cannot parse w_spec") if not space.isinstance_w(w_lst, space.w_list) or space.len_w(w_lst) < 1: raise oefmt(space.w_RuntimeError, "_commastring is not returning a list with len >= 1") @@ -544,7 +556,7 @@ if size == 1: return subdtype size *= subdtype.elsize - return W_Dtype(types.VoidType(), NPY.VOID, NPY.VOIDLTR, NPY.VOIDLTR, + return W_Dtype(types.VoidType(space), NPY.VOID, NPY.VOIDLTR, NPY.VOIDLTR, space.gettypefor(boxes.W_VoidBox), shape=shape, subdtype=subdtype, elsize=size) @@ -590,8 +602,7 @@ if w_dtype is dtype.w_box_type: return dtype if space.isinstance_w(w_dtype, space.w_type): - raise oefmt(space.w_NotImplementedError, - "cannot create dtype with type '%N'", w_dtype) + return cache.w_objectdtype raise oefmt(space.w_TypeError, "data type not understood") @@ -658,7 +669,7 @@ def new_string_dtype(space, size, char=NPY.STRINGLTR): return W_Dtype( - types.StringType(), + types.StringType(space), elsize=size, num=NPY.STRING, kind=NPY.STRINGLTR, @@ -668,7 +679,7 @@ def new_unicode_dtype(space, size): - itemtype = types.UnicodeType() + itemtype = types.UnicodeType(space) return W_Dtype( itemtype, elsize=size * itemtype.get_element_size(), @@ -681,7 +692,7 @@ def new_void_dtype(space, size): return W_Dtype( - types.VoidType(), + types.VoidType(space), elsize=size, num=NPY.VOID, kind=NPY.VOIDLTR, @@ -693,126 +704,126 @@ class DtypeCache(object): def __init__(self, space): self.w_booldtype = W_Dtype( - types.Bool(), + types.Bool(space), num=NPY.BOOL, kind=NPY.GENBOOLLTR, char=NPY.BOOLLTR, w_box_type=space.gettypefor(boxes.W_BoolBox), ) self.w_int8dtype = W_Dtype( - types.Int8(), + types.Int8(space), num=NPY.BYTE, kind=NPY.SIGNEDLTR, char=NPY.BYTELTR, w_box_type=space.gettypefor(boxes.W_Int8Box), ) self.w_uint8dtype = W_Dtype( - types.UInt8(), + types.UInt8(space), num=NPY.UBYTE, kind=NPY.UNSIGNEDLTR, char=NPY.UBYTELTR, w_box_type=space.gettypefor(boxes.W_UInt8Box), ) self.w_int16dtype = W_Dtype( - types.Int16(), + types.Int16(space), num=NPY.SHORT, kind=NPY.SIGNEDLTR, char=NPY.SHORTLTR, w_box_type=space.gettypefor(boxes.W_Int16Box), ) self.w_uint16dtype = W_Dtype( - types.UInt16(), + types.UInt16(space), num=NPY.USHORT, kind=NPY.UNSIGNEDLTR, char=NPY.USHORTLTR, w_box_type=space.gettypefor(boxes.W_UInt16Box), ) self.w_int32dtype = W_Dtype( - types.Int32(), + types.Int32(space), num=NPY.INT, kind=NPY.SIGNEDLTR, char=NPY.INTLTR, w_box_type=space.gettypefor(boxes.W_Int32Box), ) self.w_uint32dtype = W_Dtype( - types.UInt32(), + types.UInt32(space), num=NPY.UINT, kind=NPY.UNSIGNEDLTR, char=NPY.UINTLTR, w_box_type=space.gettypefor(boxes.W_UInt32Box), ) self.w_longdtype = W_Dtype( - types.Long(), + types.Long(space), num=NPY.LONG, kind=NPY.SIGNEDLTR, char=NPY.LONGLTR, w_box_type=space.gettypefor(boxes.W_LongBox), ) self.w_ulongdtype = W_Dtype( - types.ULong(), + types.ULong(space), num=NPY.ULONG, kind=NPY.UNSIGNEDLTR, char=NPY.ULONGLTR, w_box_type=space.gettypefor(boxes.W_ULongBox), ) self.w_int64dtype = W_Dtype( - types.Int64(), + types.Int64(space), num=NPY.LONGLONG, kind=NPY.SIGNEDLTR, char=NPY.LONGLONGLTR, w_box_type=space.gettypefor(boxes.W_Int64Box), ) self.w_uint64dtype = W_Dtype( - types.UInt64(), + types.UInt64(space), num=NPY.ULONGLONG, kind=NPY.UNSIGNEDLTR, char=NPY.ULONGLONGLTR, w_box_type=space.gettypefor(boxes.W_UInt64Box), ) self.w_float32dtype = W_Dtype( - types.Float32(), + types.Float32(space), num=NPY.FLOAT, kind=NPY.FLOATINGLTR, char=NPY.FLOATLTR, w_box_type=space.gettypefor(boxes.W_Float32Box), ) self.w_float64dtype = W_Dtype( - types.Float64(), + types.Float64(space), num=NPY.DOUBLE, kind=NPY.FLOATINGLTR, char=NPY.DOUBLELTR, w_box_type=space.gettypefor(boxes.W_Float64Box), ) self.w_floatlongdtype = W_Dtype( - types.FloatLong(), + types.FloatLong(space), num=NPY.LONGDOUBLE, kind=NPY.FLOATINGLTR, char=NPY.LONGDOUBLELTR, w_box_type=space.gettypefor(boxes.W_FloatLongBox), ) self.w_complex64dtype = W_Dtype( - types.Complex64(), + types.Complex64(space), num=NPY.CFLOAT, kind=NPY.COMPLEXLTR, char=NPY.CFLOATLTR, w_box_type=space.gettypefor(boxes.W_Complex64Box), ) self.w_complex128dtype = W_Dtype( - types.Complex128(), + types.Complex128(space), num=NPY.CDOUBLE, kind=NPY.COMPLEXLTR, char=NPY.CDOUBLELTR, w_box_type=space.gettypefor(boxes.W_Complex128Box), ) self.w_complexlongdtype = W_Dtype( - types.ComplexLong(), + types.ComplexLong(space), num=NPY.CLONGDOUBLE, kind=NPY.COMPLEXLTR, char=NPY.CLONGDOUBLELTR, w_box_type=space.gettypefor(boxes.W_ComplexLongBox), ) self.w_stringdtype = W_Dtype( - types.StringType(), + types.StringType(space), elsize=0, num=NPY.STRING, kind=NPY.STRINGLTR, @@ -820,7 +831,7 @@ w_box_type=space.gettypefor(boxes.W_StringBox), ) self.w_unicodedtype = W_Dtype( - types.UnicodeType(), + types.UnicodeType(space), elsize=0, num=NPY.UNICODE, kind=NPY.UNICODELTR, @@ -828,7 +839,7 @@ w_box_type=space.gettypefor(boxes.W_UnicodeBox), ) self.w_voiddtype = W_Dtype( - types.VoidType(), + types.VoidType(space), elsize=0, num=NPY.VOID, kind=NPY.VOIDLTR, @@ -836,26 +847,33 @@ w_box_type=space.gettypefor(boxes.W_VoidBox), ) self.w_float16dtype = W_Dtype( - types.Float16(), + types.Float16(space), num=NPY.HALF, kind=NPY.FLOATINGLTR, char=NPY.HALFLTR, w_box_type=space.gettypefor(boxes.W_Float16Box), ) self.w_intpdtype = W_Dtype( - types.Long(), + types.Long(space), num=NPY.LONG, kind=NPY.SIGNEDLTR, char=NPY.INTPLTR, w_box_type=space.gettypefor(boxes.W_LongBox), ) self.w_uintpdtype = W_Dtype( - types.ULong(), + types.ULong(space), num=NPY.ULONG, kind=NPY.UNSIGNEDLTR, char=NPY.UINTPLTR, w_box_type=space.gettypefor(boxes.W_ULongBox), ) + self.w_objectdtype = W_Dtype( + types.ObjectType(space), + num=NPY.OBJECT, + kind=NPY.OBJECTLTR, + char=NPY.OBJECTLTR, + w_box_type=space.gettypefor(boxes.W_ObjectBox), + ) aliases = { NPY.BOOL: ['bool_', 'bool8'], NPY.BYTE: ['byte'], @@ -874,6 +892,7 @@ NPY.CLONGDOUBLE: ['clongdouble', 'clongfloat'], NPY.STRING: ['string_', 'str'], NPY.UNICODE: ['unicode_'], + NPY.OBJECT: ['object_'], } self.alternate_constructors = { NPY.BOOL: [space.w_bool], @@ -892,6 +911,8 @@ NPY.UNICODE: [space.w_unicode], NPY.VOID: [space.gettypefor(boxes.W_GenericBox)], #space.w_buffer, # XXX no buffer in space + NPY.OBJECT: [space.gettypefor(boxes.W_ObjectBox), + space.w_object], } float_dtypes = [self.w_float16dtype, self.w_float32dtype, self.w_float64dtype, self.w_floatlongdtype] @@ -911,7 +932,7 @@ self.w_int64dtype, self.w_uint64dtype, ] + float_dtypes + complex_dtypes + [ self.w_stringdtype, self.w_unicodedtype, self.w_voiddtype, - self.w_intpdtype, self.w_uintpdtype, + self.w_intpdtype, self.w_uintpdtype, self.w_objectdtype, ] self.float_dtypes_by_num_bytes = sorted( (dtype.elsize, dtype) @@ -963,6 +984,7 @@ 'USHORT': self.w_uint16dtype, 'FLOAT': self.w_float32dtype, 'BOOL': self.w_booldtype, + 'OBJECT': self.w_objectdtype, } typeinfo_partial = { diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -202,11 +202,16 @@ return self elif isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool() \ and w_idx.ndims() > 0: - return self.getitem_filter(space, w_idx) - try: - return self.implementation.descr_getitem(space, self, w_idx) - except ArrayArgumentException: - return self.getitem_array_int(space, w_idx) + w_ret = self.getitem_filter(space, w_idx) + else: + try: + w_ret = self.implementation.descr_getitem(space, self, w_idx) + except ArrayArgumentException: + w_ret = self.getitem_array_int(space, w_idx) + if isinstance(w_ret, boxes.W_ObjectBox): + #return the W_Root object, not a scalar + w_ret = w_ret.w_obj + return w_ret def getitem(self, space, index_list): return self.implementation.getitem_index(space, index_list) @@ -550,6 +555,7 @@ else: strides = self.descr_get_strides(space) space.setitem_str(w_d, 'strides', strides) + space.setitem_str(w_d, 'version', space.wrap(3)) return w_d w_pypy_data = None @@ -845,7 +851,7 @@ "new type not compatible with array.")) # Strides, shape does not change v = impl.astype(space, dtype) - return wrap_impl(space, w_type, self, v) + return wrap_impl(space, w_type, self, v) strides = impl.get_strides() if dims == 1 or strides[0] = 0 - else: - a = array(Polynomial()) - assert a.shape == () + diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -17,6 +17,7 @@ def __init__(self): self.base = self self.elsize = 1 + self.num = 0 def create_slice(space, a, chunks): @@ -2182,7 +2183,8 @@ assert b.dtype == 'bool' a = arange(6, dtype='f4').reshape(2,3) - b = a.astype('i4') + b = a.T.astype('i4') + assert (a.T.strides == b.strides) a = array('x').astype('S3').dtype assert a.itemsize == 3 @@ -3150,11 +3152,7 @@ assert b[35] == 200 b[[slice(25, 30)]] = range(5) assert all(a[:5] == range(5)) - import sys - if '__pypy__' not in sys.builtin_module_names: - raises(TypeError, 'b[[[slice(25, 125)]]]') - else: - raises(NotImplementedError, 'b[[[slice(25, 125)]]]') + raises(IndexError, 'b[[[slice(25, 125)]]]') def test_cumsum(self): from numpy import arange diff --git a/pypy/module/micronumpy/test/test_object_arrays.py b/pypy/module/micronumpy/test/test_object_arrays.py new file mode 100644 --- /dev/null +++ b/pypy/module/micronumpy/test/test_object_arrays.py @@ -0,0 +1,162 @@ +from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest + + +class AppTestObjectDtypes(BaseNumpyAppTest): + def test_scalar_from_object(self): + from numpy import array + import sys + class Polynomial(object): + def whatami(self): + return 'an object' + a = array(Polynomial()) + assert a.shape == () + assert a.sum().whatami() == 'an object' + + def test_uninitialized_object_array_is_filled_by_None(self): + import numpy as np + + a = np.ndarray([5], dtype="O") + + assert a[0] == None + + def test_object_arrays_add(self): + import numpy as np + + a = np.array(["foo"], dtype=object) + b = np.array(["bar"], dtype=object) + raises(TypeError, np.add, a, 1) + res = a + b + assert res[0] == "foobar" + + def test_bool_func(self): + import numpy as np + a = np.array(["foo"], dtype=object) + b = a and complex(1, -1) + assert b == complex(1, -1) + b = np.array(complex(1, -1)) and a + assert (b == a).all() + c = np.array([1, 2, 3]) + assert (a[0] != c[0]) + assert (c[0] != a[0]) + assert (a[0] > c[0]) + assert (not a[0] < c[0]) + assert (c[0] < a[0]) + assert (not c[0] > a[0]) + + def test_logical_ufunc(self): + import numpy as np + import sys + + if '__pypy__' in sys.builtin_module_names: + skip('need to refactor use of raw_xxx_op in types to make this work') + a = np.array(["foo"], dtype=object) + b = np.array([1], dtype=object) + d = np.array([complex(1, 10)], dtype=object) + c = np.logical_and(a, 1) + assert c.dtype == np.dtype('object') + assert c == 1 + c = np.logical_and(b, complex(1, -1)) + assert c.dtype == np.dtype('object') + assert c == complex(1, -1) + c = np.logical_and(d, b) + assert c == 1 + c = b & 1 + assert c.dtype == np.dtype('object') + assert (c == 1).all() + c = np.array(1) & b + assert (c == b).all() + + def test_reduce(self): + import numpy as np + class O(object): + def whatami(self): + return 'an object' + fiveOs = [O()] * 5 + a = np.array(fiveOs, dtype=object) + print np.maximum + b = np.maximum.reduce(a) + assert b is not None + + def test_complex_op(self): + import numpy as np + import sys + a = np.array(['abc', 'def'], dtype=object) + b = np.array([1, 2, 3], dtype=object) + c = np.array([complex(1, 1), complex(1, -1)], dtype=object) + for arg in (a,b,c): + assert (arg == np.real(arg)).all() + assert (0 == np.imag(arg)).all() + if '__pypy__' in sys.builtin_module_names: + skip('not implemented yet') + raises(AttributeError, np.conj, a) + res = np.conj(b) + assert (res == b).all() + res = np.conj(c) + assert res[0] == c[1] and res[1] == c[0] + + def test_keep_object_alive(self): + # only translated does it really test the gc + import numpy as np + import gc + class O(object): + def whatami(self): + return 'an object' + fiveOs = [O()] * 5 + a = np.array(fiveOs, dtype=object) + del fiveOs + gc.collect() + assert a[2].whatami() == 'an object' + + def test_array_interface(self): + import numpy as np + import sys + class DummyArray(object): + def __init__(self, interface, base=None): + self.__array_interface__ = interface + self.base = base + a = np.array([(1, 2, 3)], dtype='u4,u4,u4') + b = np.array([(1, 2, 3), (4, 5, 6), (7, 8, 9)], dtype='u4,u4,u4') + interface = dict(a.__array_interface__) + interface['shape'] = tuple([3]) + interface['strides'] = tuple([0]) + if '__pypy__' in sys.builtin_module_names: + skip('not implemented yet') + c = np.array(DummyArray(interface, base=a)) + c.dtype = a.dtype + #print c + assert (c == np.array([(1, 2, 3), (1, 2, 3), (1, 2, 3)], dtype='u4,u4,u4') ).all() + + def test_for_object_scalar_creation(self): + import numpy as np + import sys + a = np.object_() + b = np.object_(3) + b2 = np.object_(3.0) + c = np.object_([4, 5]) + d = np.array([None])[0] + assert a is None + assert type(b) is int + assert type(b2) is float + assert type(c) is np.ndarray + assert c.dtype == object + assert type(d) is type(None) + if '__pypy__' in sys.builtin_module_names: + skip('not implemented yet') + e = np.object_([None, {}, []]) + assert e.dtype == object + + def test_mem_array_creation_invalid_specification(self): + # while not specifically testing object dtype, this + # test segfaulted during ObjectType.store due to + # missing gc hooks + import numpy as np + import sys + ytype = np.object_ + if '__pypy__' in sys.builtin_module_names: + ytype = str + dt = np.dtype([('x', int), ('y', ytype)]) + # Correct way + a = np.array([(1, 'object')], dt) + # Wrong way - should complain about writing buffer to object dtype + raises(ValueError, np.array, [1, 'object'], dt) + diff --git a/pypy/module/micronumpy/test/test_selection.py b/pypy/module/micronumpy/test/test_selection.py --- a/pypy/module/micronumpy/test/test_selection.py +++ b/pypy/module/micronumpy/test/test_selection.py @@ -12,14 +12,11 @@ exp = sorted(range(len(exp)), key=exp.__getitem__) c = a.copy() res = a.argsort() - assert (res == exp).all(), '%r\n%r\n%r' % (a,res,exp) + assert (res == exp).all(), 'Failed sortng %r\na=%r\nres=%r\nexp=%r' % (dtype,a,res,exp) assert (a == c).all() # not modified a = arange(100, dtype=dtype) assert (a.argsort() == a).all() - import sys - if '__pypy__' in sys.builtin_module_names: - raises(NotImplementedError, 'arange(10,dtype="float16").argsort()') def test_argsort_ndim(self): from numpy import array @@ -63,14 +60,13 @@ 'i2', complex]: a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype) exp = sorted(list(a)) - res = a.copy() - res.sort() - assert (res == exp).all(), '%r\n%r\n%r' % (a,res,exp) + a.sort() + assert (a == exp).all(), 'Failed sorting %r\n%r\n%r' % (dtype, a, exp) a = arange(100, dtype=dtype) c = a.copy() a.sort() - assert (a == c).all() + assert (a == c).all(), 'Failed sortng %r\na=%r\nc=%r' % (dtype,a,c) def test_sort_nonnative(self): from numpy import array @@ -222,6 +218,7 @@ def test_sort_objects(self): # test object array sorts. + skip('object type not supported yet') from numpy import empty try: a = empty((101,), dtype=object) diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py --- a/pypy/module/micronumpy/test/test_ufuncs.py +++ b/pypy/module/micronumpy/test/test_ufuncs.py @@ -397,11 +397,11 @@ for i in range(3): assert min_c_b[i] == min(b[i], c) - def test_scalar(self): + def test_all_available(self): # tests that by calling all available ufuncs on scalars, none will # raise uncaught interp-level exceptions, (and crash the test) # and those that are uncallable can be accounted for. - # test on the four base-class dtypes: int, bool, float, complex + # test on the base-class dtypes: int, bool, float, complex, object # We need this test since they have no common base class. import numpy as np def find_uncallable_ufuncs(dtype): @@ -412,6 +412,11 @@ if isinstance(u, np.ufunc): try: u(* [array] * u.nin) + except AttributeError: + pass + except NotImplementedError: + print s + uncallable.add(s) except TypeError: assert s not in uncallable uncallable.add(s) @@ -427,6 +432,9 @@ 'fabs', 'fmod', 'invert', 'mod', 'logaddexp', 'logaddexp2', 'left_shift', 'right_shift', 'copysign', 'signbit', 'ceil', 'floor', 'trunc']) + assert find_uncallable_ufuncs('object') == set( + ['isnan', 'logaddexp2', 'copysign', 'isfinite', 'signbit', + 'isinf', 'logaddexp']) def test_int_only(self): from numpy import bitwise_and, array diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -9,6 +9,7 @@ from pypy.module.micronumpy.compile import FakeSpace, Parser, InterpreterState from pypy.module.micronumpy.base import W_NDimArray +py.test.skip('move these to pypyjit/test_pypy_c/test_micronumpy') class TestNumpyJit(LLJitMixin): graph = None diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -3,8 +3,9 @@ from pypy.interpreter.error import OperationError, oefmt from pypy.objspace.std.floatobject import float2string from pypy.objspace.std.complexobject import str_format +from pypy.interpreter.baseobjspace import W_Root, ObjSpace from rpython.rlib import clibffi, jit, rfloat, rcomplex -from rpython.rlib.objectmodel import specialize +from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rlib.rarithmetic import widen, byteswap, r_ulonglong, \ most_neg_value_of, LONG_BIT from rpython.rlib.rawstorage import (alloc_raw_storage, @@ -14,10 +15,12 @@ pack_float80, unpack_float80) from rpython.rlib.rstruct.nativefmttable import native_is_bigendian from rpython.rlib.rstruct.runpack import runpack -from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rtyper.annlowlevel import cast_instance_to_gcref,\ + cast_gcref_to_instance +from rpython.rtyper.lltypesystem import lltype, rffi, llmemory from rpython.tool.sourcetools import func_with_new_name from pypy.module.micronumpy import boxes -from pypy.module.micronumpy.concrete import SliceArray, VoidBoxStorage +from pypy.module.micronumpy.concrete import SliceArray, VoidBoxStorage, V_OBJECTSTORE from pypy.module.micronumpy.strides import calc_strides degToRad = math.pi / 180.0 @@ -109,10 +112,12 @@ return dispatcher class BaseType(object): - _immutable_fields_ = ['native'] + _immutable_fields_ = ['native', 'space'] - def __init__(self, native=True): + def __init__(self, space, native=True): + assert isinstance(space, ObjSpace) self.native = native + self.space = space def __repr__(self): return self.__class__.__name__ @@ -191,7 +196,7 @@ with arr as storage: self._write(storage, i, offset, self.unbox(box)) - def fill(self, storage, width, box, start, stop, offset): + def fill(self, storage, width, box, start, stop, offset, gcstruct): value = self.unbox(box) for i in xrange(start, stop, width): self._write(storage, i, offset, value) @@ -306,7 +311,7 @@ @raw_unary_op def rint(self, v): - float64 = Float64() + float64 = Float64(self.space) return float64.rint(float64.box(v)) class Bool(BaseType, Primitive): @@ -399,7 +404,7 @@ def round(self, v, decimals=0): if decimals != 0: return v - return Float64().box(self.unbox(v)) + return Float64(self.space).box(self.unbox(v)) class Integer(Primitive): _mixin_ = True @@ -444,7 +449,7 @@ self.T is rffi.LONG or self.T is rffi.LONGLONG): if v2 == -1 and v1 == self.for_computation(most_neg_value_of(self.T)): return self.box(0) - return self.box(v1 // v2) + return self.box(v1 / v2) @simple_binary_op def mod(self, v1, v2): @@ -1152,7 +1157,7 @@ with arr as storage: self._write(storage, i, offset, self.unbox(box)) - def fill(self, storage, width, box, start, stop, offset): + def fill(self, storage, width, box, start, stop, offset, gcstruct): value = self.unbox(box) for i in xrange(start, stop, width): self._write(storage, i, offset, value) @@ -1253,25 +1258,25 @@ def ge(self, v1, v2): return self._lt(v2, v1) or self._eq(v2, v1) - def _bool(self, v): + def _cbool(self, v): return bool(v[0]) or bool(v[1]) @raw_binary_op def logical_and(self, v1, v2): - return self._bool(v1) and self._bool(v2) + return self._cbool(v1) and self._cbool(v2) @raw_binary_op def logical_or(self, v1, v2): - return self._bool(v1) or self._bool(v2) + return self._cbool(v1) or self._cbool(v2) @raw_unary_op def logical_not(self, v): - return not self._bool(v) + return not self._cbool(v) @raw_binary_op def logical_xor(self, v1, v2): - a = self._bool(v1) - b = self._bool(v2) + a = self._cbool(v1) + b = self._cbool(v2) return (not b and a) or (not a and b) def min(self, v1, v2): @@ -1629,6 +1634,283 @@ BoxType = boxes.W_ComplexLongBox ComponentBoxType = boxes.W_FloatLongBox +_all_objs_for_tests = [] # for tests + +class ObjectType(Primitive, BaseType): + T = lltype.Signed + BoxType = boxes.W_ObjectBox + + def get_element_size(self): + return rffi.sizeof(lltype.Signed) + + def coerce(self, space, dtype, w_item): + if isinstance(w_item, boxes.W_ObjectBox): + return w_item + return boxes.W_ObjectBox(w_item) + + def coerce_subtype(self, space, w_subtype, w_item): + # return the item itself + return self.unbox(self.box(w_item)) + + def store(self, arr, i, offset, box): + if arr.gcstruct is V_OBJECTSTORE: + raise oefmt(self.space.w_NotImplementedError, + "cannot store object in array with no gc hook") + self._write(arr.storage, i, offset, self.unbox(box), + arr.gcstruct) + + def read(self, arr, i, offset, dtype=None): + return self.box(self._read(arr.storage, i, offset)) + + def byteswap(self, w_v): + return w_v + + @jit.dont_look_inside + def _write(self, storage, i, offset, w_obj, gcstruct): + # no GC anywhere in this function! + if we_are_translated(): + from rpython.rlib import rgc + rgc.ll_writebarrier(gcstruct) + value = rffi.cast(lltype.Signed, cast_instance_to_gcref(w_obj)) + else: + value = len(_all_objs_for_tests) + _all_objs_for_tests.append(w_obj) + raw_storage_setitem_unaligned(storage, i + offset, value) + + @jit.dont_look_inside + def _read(self, storage, i, offset): + res = raw_storage_getitem_unaligned(self.T, storage, i + offset) + if we_are_translated(): + gcref = rffi.cast(llmemory.GCREF, res) + w_obj = cast_gcref_to_instance(W_Root, gcref) + else: + w_obj = _all_objs_for_tests[res] + return w_obj + + def fill(self, storage, width, box, start, stop, offset, gcstruct): + value = self.unbox(box) + for i in xrange(start, stop, width): + self._write(storage, i, offset, value, gcstruct) + + def unbox(self, box): + if isinstance(box, self.BoxType): + return box.w_obj + else: + raise oefmt(self.space.w_NotImplementedError, + "object dtype cannot unbox %s", str(box)) + + @specialize.argtype(1) + def box(self, w_obj): + if isinstance(w_obj, W_Root): + pass + elif isinstance(w_obj, bool): + w_obj = self.space.newbool(w_obj) + elif isinstance(w_obj, int): + w_obj = self.space.newint(w_obj) + elif isinstance(w_obj, lltype.Number): + w_obj = self.space.newint(w_obj) + elif isinstance(w_obj, float): + w_obj = self.space.newfloat(w_obj) + elif w_obj is None: + w_obj = self.space.w_None + else: + raise oefmt(self.space.w_NotImplementedError, + "cannot create object array/scalar from lltype") + return self.BoxType(w_obj) + + @specialize.argtype(1, 2) + def box_complex(self, real, imag): + if isinstance(real, rffi.r_singlefloat): + real = rffi.cast(rffi.DOUBLE, real) + if isinstance(imag, rffi.r_singlefloat): + imag = rffi.cast(rffi.DOUBLE, imag) + w_obj = self.space.newcomplex(real, imag) + return self.BoxType(w_obj) + + def str_format(self, box): + return self.space.str_w(self.space.repr(self.unbox(box))) + + def runpack_str(self, space, s): + raise oefmt(space.w_NotImplementedError, + "fromstring not implemented for object type") + + def to_builtin_type(self, space, box): + assert isinstance(box, self.BoxType) + return box.w_obj + + @staticmethod + def for_computation(v): + return v + + @raw_binary_op + def eq(self, v1, v2): + return self.space.eq_w(v1, v2) + + @simple_binary_op + def max(self, v1, v2): + if self.space.is_true(self.space.ge(v1, v2)): + return v1 + return v2 + + @simple_binary_op + def min(self, v1, v2): + if self.space.is_true(self.space.le(v1, v2)): + return v1 + return v2 + + @raw_unary_op + def bool(self,v): + return self._obool(v) + + def _obool(self, v): + if self.space.is_true(v): + return True + return False + + @raw_binary_op + def logical_and(self, v1, v2): + if self._obool(v1): + return self.space.bool_w(v2) + return self.space.bool_w(v1) + + @raw_binary_op + def logical_or(self, v1, v2): + if self._obool(v1): + return self.space.bool_w(v1) + return self.space.bool_w(v2) + + @raw_unary_op + def logical_not(self, v): + return not self._obool(v) + + @raw_binary_op + def logical_xor(self, v1, v2): + a = self._obool(v1) + b = self._obool(v2) + return (not b and a) or (not a and b) + + @simple_binary_op + def bitwise_and(self, v1, v2): + return self.space.and_(v1, v2) + + @simple_binary_op + def bitwise_or(self, v1, v2): + return self.space.or_(v1, v2) + + @simple_binary_op + def bitwise_xor(self, v1, v2): + return self.space.xor(v1, v2) + + @simple_binary_op + def pow(self, v1, v2): + return self.space.pow(v1, v2, self.space.wrap(1)) + + @simple_unary_op + def reciprocal(self, v1): + return self.space.div(self.space.wrap(1.0), v1) + + @simple_unary_op + def sign(self, v): + zero = self.space.wrap(0) + one = self.space.wrap(1) + m_one = self.space.wrap(-1) + if self.space.is_true(self.space.gt(v, zero)): + return one + elif self.space.is_true(self.space.lt(v, zero)): + return m_one + else: + return zero + + @simple_unary_op + def real(self, v): + return v + + @simple_unary_op + def imag(self, v): + return 0 + + @simple_unary_op + def square(self, v): + return self.space.mul(v, v) + + @raw_binary_op + def le(self, v1, v2): + return self.space.bool_w(self.space.le(v1, v2)) + + @raw_binary_op + def ge(self, v1, v2): + return self.space.bool_w(self.space.ge(v1, v2)) + + @raw_binary_op + def lt(self, v1, v2): + return self.space.bool_w(self.space.lt(v1, v2)) + + @raw_binary_op + def gt(self, v1, v2): + return self.space.bool_w(self.space.gt(v1, v2)) + + @raw_binary_op + def ne(self, v1, v2): + return self.space.bool_w(self.space.ne(v1, v2)) + +def add_attributeerr_op(cls, op): + def func(self, *args): + raise oefmt(self.space.w_AttributeError, + "%s", op) + func.__name__ = 'object_' + op + setattr(cls, op, func) + +def add_unsupported_op(cls, op): + def func(self, *args): + raise oefmt(self.space.w_TypeError, + "ufunc '%s' not supported for input types", op) + func.__name__ = 'object_' + op + setattr(cls, op, func) + +def add_unary_op(cls, op, method): + @simple_unary_op + def func(self, w_v): + space = self.space + w_impl = space.lookup(w_v, method) + if w_impl is None: + raise oefmt(space.w_AttributeError, 'unknown op "%s" on object' % op) + return space.get_and_call_function(w_impl, w_v) + func.__name__ = 'object_' + op + setattr(cls, op, func) + +def add_space_unary_op(cls, op): + @simple_unary_op + def func(self, v): + return getattr(self.space, op)(v) + func.__name__ = 'object_' + op + setattr(cls, op, func) + +def add_space_binary_op(cls, op): + @simple_binary_op + def func(self, v1, v2): + return getattr(self.space, op)(v1, v2) + func.__name__ = 'object_' + op + setattr(cls, op, func) + +for op in ('copysign', 'isfinite', 'isinf', 'isnan', 'logaddexp', 'logaddexp2', + 'signbit'): + add_unsupported_op(ObjectType, op) +for op in ('arctan2', 'arccos', 'arccosh', 'arcsin', 'arcsinh', 'arctan', + 'arctanh', 'ceil', 'floor', 'cos', 'sin', 'tan', 'cosh', 'sinh', + 'tanh', 'radians', 'degrees', 'exp','exp2', 'expm1', 'fabs', + 'log', 'log10', 'log1p', 'log2', 'sqrt', 'trunc'): + add_attributeerr_op(ObjectType, op) +for op in ('abs', 'neg', 'pos', 'invert'): + add_space_unary_op(ObjectType, op) +for op, method in (('conj', 'descr_conjugate'), ('rint', 'descr_rint')): + add_unary_op(ObjectType, op, method) +for op in ('add', 'floordiv', 'div', 'mod', 'mul', 'sub', 'lshift', 'rshift'): + add_space_binary_op(ObjectType, op) + +ObjectType.fmax = ObjectType.max +ObjectType.fmin = ObjectType.min +ObjectType.fmod = ObjectType.mod + class FlexibleType(BaseType): def get_element_size(self): return rffi.sizeof(self.T) @@ -1758,7 +2040,7 @@ def bool(self, v): return bool(self.to_str(v)) - def fill(self, storage, width, box, start, stop, offset): + def fill(self, storage, width, box, start, stop, offset, gcstruct): for i in xrange(start, stop, width): self._store(storage, i, offset, box, width) @@ -1775,6 +2057,57 @@ raise OperationError(space.w_NotImplementedError, space.wrap( "coerce (probably from set_item) not implemented for unicode type")) + def store(self, arr, i, offset, box): + assert isinstance(box, boxes.W_UnicodeBox) + raise oefmt(self.space.w_NotImplementedError, "unicode type not completed") + + def read(self, arr, i, offset, dtype=None): + raise oefmt(self.space.w_NotImplementedError, "unicode type not completed") + + def str_format(self, item): + raise oefmt(self.space.w_NotImplementedError, "unicode type not completed") + + def to_builtin_type(self, space, box): + raise oefmt(self.space.w_NotImplementedError, "unicode type not completed") + + def eq(self, v1, v2): + raise oefmt(self.space.w_NotImplementedError, "unicode type not completed") + + def ne(self, v1, v2): + raise oefmt(self.space.w_NotImplementedError, "unicode type not completed") + + def lt(self, v1, v2): + raise oefmt(self.space.w_NotImplementedError, "unicode type not completed") + + def le(self, v1, v2): + raise oefmt(self.space.w_NotImplementedError, "unicode type not completed") + + def gt(self, v1, v2): + raise oefmt(self.space.w_NotImplementedError, "unicode type not completed") + + def ge(self, v1, v2): + raise oefmt(self.space.w_NotImplementedError, "unicode type not completed") + + def logical_and(self, v1, v2): + raise oefmt(self.space.w_NotImplementedError, "unicode type not completed") + + def logical_or(self, v1, v2): + raise oefmt(self.space.w_NotImplementedError, "unicode type not completed") + + def logical_not(self, v): + raise oefmt(self.space.w_NotImplementedError, "unicode type not completed") + + @str_binary_op + def logical_xor(self, v1, v2): + raise oefmt(self.space.w_NotImplementedError, "unicode type not completed") + + def bool(self, v): + raise oefmt(self.space.w_NotImplementedError, "unicode type not completed") + + def fill(self, storage, width, box, start, stop, offset, gcstruct): + raise oefmt(self.space.w_NotImplementedError, "unicode type not completed") + + class VoidType(FlexibleType): T = lltype.Char @@ -1882,6 +2215,9 @@ items_w = space.fixedview(w_item) elif isinstance(w_item, W_NDimArray) and w_item.is_scalar(): items_w = space.fixedview(w_item.get_scalar_value()) + elif space.isinstance_w(w_item, space.w_list): + raise oefmt(space.w_TypeError, + "expected a readable buffer object") else: # XXX support initializing from readable buffers items_w = [w_item] * len(dtype.fields) @@ -1913,7 +2249,7 @@ for k in range(size): storage[k + i + ofs] = box_storage[k + box.ofs] - def fill(self, storage, width, box, start, stop, offset): + def fill(self, storage, width, box, start, stop, offset, gcstruct): assert isinstance(box, boxes.W_VoidBox) assert width == box.dtype.elsize for i in xrange(start, stop, width): diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py --- a/pypy/module/micronumpy/ufuncs.py +++ b/pypy/module/micronumpy/ufuncs.py @@ -349,7 +349,7 @@ if dtype.is_flexible(): raise OperationError(space.w_TypeError, space.wrap('Not implemented for this type')) - if (self.int_only and not dtype.is_int() or + if (self.int_only and not (dtype.is_int() or dtype.is_object()) or not self.allow_bool and dtype.is_bool() or not self.allow_complex and dtype.is_complex()): raise oefmt(space.w_TypeError, @@ -378,6 +378,8 @@ w_val = self.func(calc_dtype, w_obj.get_scalar_value().convert_to(space, calc_dtype)) if out is None: + if res_dtype.is_object(): + w_val = w_obj.get_scalar_value() return w_val w_val = res_dtype.coerce(space, w_val) if out.is_scalar(): @@ -434,11 +436,20 @@ w_rhs = numpify(space, w_rhs) w_ldtype = _get_dtype(space, w_lhs) w_rdtype = _get_dtype(space, w_rhs) - if w_ldtype.is_str() and w_rdtype.is_str() and \ + if w_ldtype.is_object() or w_rdtype.is_object(): + pass + elif w_ldtype.is_str() and w_rdtype.is_str() and \ self.comparison_func: pass - elif (w_ldtype.is_str() or w_rdtype.is_str()) and \ + elif (w_ldtype.is_str()) and \ self.comparison_func and w_out is None: + if self.name in ('equal', 'less_equal', 'less'): + return space.wrap(False) + return space.wrap(True) + elif (w_rdtype.is_str()) and \ + self.comparison_func and w_out is None: + if self.name in ('not_equal','less', 'less_equal'): + return space.wrap(True) return space.wrap(False) elif w_ldtype.is_flexible() or w_rdtype.is_flexible(): if self.comparison_func: @@ -463,9 +474,9 @@ w_ldtype, w_rdtype, promote_to_float=self.promote_to_float, promote_bools=self.promote_bools) - if (self.int_only and (not w_ldtype.is_int() or - not w_rdtype.is_int() or - not calc_dtype.is_int()) or + if (self.int_only and (not (w_ldtype.is_int() or w_ldtype.is_object()) or + not (w_rdtype.is_int() or w_rdtype.is_object()) or + not (calc_dtype.is_int() or calc_dtype.is_object())) or not self.allow_bool and (w_ldtype.is_bool() or w_rdtype.is_bool()) or not self.allow_complex and (w_ldtype.is_complex() or @@ -643,7 +654,7 @@ # from frompyfunc pass # mimic NpyIter_AdvancedNew with a nditer - w_itershape = space.newlist([space.wrap(i) for i in iter_shape]) + w_itershape = space.newlist([space.wrap(i) for i in iter_shape]) nd_it = W_NDIter(space, space.newlist(inargs + outargs), w_flags, w_op_flags, w_op_dtypes, w_casting, w_op_axes, w_itershape) @@ -749,7 +760,7 @@ else: raise oefmt(space.w_TypeError, "a type-string for %s " \ "requires 1 typecode or %d typecode(s) before and %d" \ - " after the -> sign, not '%s'", self.name, self.nin, + " after the -> sign, not '%s'", self.name, self.nin, self.nout, type_tup) except KeyError: raise oefmt(space.w_ValueError, "unknown typecode in" \ @@ -773,11 +784,11 @@ for j in range(self.nargs): if dtypes[j] is not None and dtypes[j] != _dtypes[i+j]: allok = False - if allok: + if allok: break else: if len(self.funcs) > 1: - + dtypesstr = '' for d in dtypes: if d is None: @@ -787,7 +798,7 @@ _dtypesstr = ','.join(['%s%s%s' % (d.byteorder, d.kind, d.elsize) \ for d in _dtypes]) raise oefmt(space.w_TypeError, - "input dtype [%s] did not match any known dtypes [%s] ", + "input dtype [%s] did not match any known dtypes [%s] ", dtypesstr,_dtypesstr) i = 0 # Fill in empty dtypes @@ -807,7 +818,7 @@ assert isinstance(curarg, W_NDimArray) if len(arg_shapes[i]) != curarg.ndims(): # reshape - + sz = product(curarg.get_shape()) * curarg.get_dtype().elsize with curarg.implementation as storage: inargs[i] = W_NDimArray.from_shape_and_storage( @@ -865,7 +876,7 @@ "%s of gufunc was not specified", self.name, name, _i, core_dim_index, self.signature) target_dims.append(v) - arg_shapes.append(iter_shape + target_dims) + arg_shapes.append(iter_shape + target_dims) continue n = len(curarg.get_shape()) - num_dims if n < 0: @@ -907,7 +918,7 @@ raise oefmt(space.w_ValueError, "%s: %s operand %d has a " "mismatch in its core dimension %d, with gufunc " "signature %s (expected %d, got %d)", - self.name, name, _i, j, + self.name, name, _i, j, self.signature, matched_dims[core_dim_index], dims_to_match[core_dim_index]) #print 'adding',iter_shape,'+',dims_to_match,'to arg_shapes' @@ -950,6 +961,10 @@ return dt1 if dt1 is None: return dt2 + + if dt1.num == NPY.OBJECT or dt2.num == NPY.OBJECT: + return get_dtype_cache(space).w_objectdtype + # dt1.num should be <= dt2.num if dt1.num > dt2.num: dt1, dt2 = dt2, dt1 @@ -1032,6 +1047,8 @@ @jit.unroll_safe def find_unaryop_result_dtype(space, dt, promote_to_float=False, promote_bools=False, promote_to_largest=False): + if dt.is_object(): + return dt if promote_to_largest: if dt.kind == NPY.GENBOOLLTR or dt.kind == NPY.SIGNEDLTR: if dt.elsize * 8 < LONG_BIT: @@ -1064,6 +1081,7 @@ uint64_dtype = get_dtype_cache(space).w_uint64dtype complex_dtype = get_dtype_cache(space).w_complex128dtype float_dtype = get_dtype_cache(space).w_float64dtype + object_dtype = get_dtype_cache(space).w_objectdtype if isinstance(w_obj, boxes.W_GenericBox): dtype = w_obj.get_dtype(space) return find_binop_result_dtype(space, dtype, current_guess) @@ -1097,9 +1115,10 @@ return variable_dtype(space, 'S%d' % space.len_w(w_obj)) return current_guess - raise oefmt(space.w_NotImplementedError, - 'unable to create dtype from objects, "%T" instance not ' - 'supported', w_obj) + return object_dtype + #raise oefmt(space.w_NotImplementedError, + # 'unable to create dtype from objects, "%T" instance not ' + # 'supported', w_obj) def ufunc_dtype_caller(space, ufunc_name, op_name, nin, comparison_func, @@ -1263,7 +1282,7 @@ w_identity=None, name='', doc='', stack_inputs=False): ''' frompyfunc(func, nin, nout) #cpython numpy compatible frompyfunc(func, nin, nout, dtypes=None, signature='', - identity=None, name='', doc='', + identity=None, name='', doc='', stack_inputs=False) Takes an arbitrary Python function and returns a ufunc. @@ -1282,7 +1301,7 @@ dtypes: None or [dtype, ...] of the input, output args for each function, or 'match' to force output to exactly match input dtype Note that 'match' is a pypy-only extension to allow non-object - return dtypes + return dtypes signature*: str, default='' The mapping of input args to output args, defining the inner-loop indexing. If it is empty, the func operates on scalars @@ -1293,7 +1312,7 @@ stack_inputs*: boolean, whether the function is of the form out = func(*in) False or - func(*[in + out]) True + func(*[in + out]) True only one of out_dtype or signature may be specified diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py --- a/pypy/tool/pytest/appsupport.py +++ b/pypy/tool/pytest/appsupport.py @@ -99,6 +99,7 @@ debug_excs = getattr(operr, 'debug_excs', []) if debug_excs: self._excinfo = debug_excs[0] + self.value = self.operr.errorstr(self.space) # XXX def __repr__(self): return "" % self.operr.errorstr(self.space) diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -686,6 +686,15 @@ lambda_customtrace = lambda: customtrace """ + at specialize.ll() +def ll_writebarrier(gc_obj): + """Use together with custom tracers. When you update some object pointer + stored in raw memory, you must call this function on 'gc_obj', which must + be the object of type TP with the custom tracer (*not* the value stored!). + This makes sure that the custom hook will be called again.""" + from rpython.rtyper.lltypesystem.lloperation import llop + llop.gc_writebarrier(lltype.Void, gc_obj) + class RegisterGcTraceEntry(ExtRegistryEntry): _about_ = register_custom_trace_hook From noreply at buildbot.pypy.org Fri Apr 24 07:00:45 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Fri, 24 Apr 2015 07:00:45 +0200 (CEST) Subject: [pypy-commit] pypy can_cast: remove (almost) unused w_intpdtype and w_uintpdtype Message-ID: <20150424050045.52A231C069E@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: can_cast Changeset: r76912:c009ce07c90f Date: 2015-04-24 04:17 +0100 http://bitbucket.org/pypy/pypy/changeset/c009ce07c90f/ Log: remove (almost) unused w_intpdtype and w_uintpdtype diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py --- a/pypy/module/micronumpy/descriptor.py +++ b/pypy/module/micronumpy/descriptor.py @@ -853,20 +853,6 @@ char=NPY.HALFLTR, w_box_type=space.gettypefor(boxes.W_Float16Box), ) - self.w_intpdtype = W_Dtype( - types.Long(space), - num=NPY.LONG, - kind=NPY.SIGNEDLTR, - char=NPY.INTPLTR, - w_box_type=space.gettypefor(boxes.W_LongBox), - ) - self.w_uintpdtype = W_Dtype( - types.ULong(space), - num=NPY.ULONG, - kind=NPY.UNSIGNEDLTR, - char=NPY.UINTPLTR, - w_box_type=space.gettypefor(boxes.W_ULongBox), - ) self.w_objectdtype = W_Dtype( types.ObjectType(space), num=NPY.OBJECT, @@ -932,7 +918,7 @@ self.w_int64dtype, self.w_uint64dtype, ] + float_dtypes + complex_dtypes + [ self.w_stringdtype, self.w_unicodedtype, self.w_voiddtype, - self.w_intpdtype, self.w_uintpdtype, self.w_objectdtype, + self.w_objectdtype, ] self.float_dtypes_by_num_bytes = sorted( (dtype.elsize, dtype) @@ -973,8 +959,7 @@ 'CLONGDOUBLE': self.w_complexlongdtype, #'DATETIME', 'UINT': self.w_uint32dtype, - 'INTP': self.w_intpdtype, - 'UINTP': self.w_uintpdtype, + 'INTP': self.w_longdtype, 'HALF': self.w_float16dtype, 'BYTE': self.w_int8dtype, #'TIMEDELTA', From noreply at buildbot.pypy.org Fri Apr 24 07:00:46 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Fri, 24 Apr 2015 07:00:46 +0200 (CEST) Subject: [pypy-commit] pypy can_cast: Add num, kind and char information to the itemtypes Message-ID: <20150424050046.A6A011C069E@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: can_cast Changeset: r76913:0efb8027fc48 Date: 2015-04-24 04:18 +0100 http://bitbucket.org/pypy/pypy/changeset/0efb8027fc48/ Log: Add num, kind and char information to the itemtypes diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -22,6 +22,7 @@ from pypy.module.micronumpy import boxes from pypy.module.micronumpy.concrete import SliceArray, VoidBoxStorage, V_OBJECTSTORE from pypy.module.micronumpy.strides import calc_strides +from . import constants as NPY degToRad = math.pi / 180.0 log2 = math.log(2) @@ -316,6 +317,9 @@ class Bool(BaseType, Primitive): T = lltype.Bool + num = NPY.BOOL + kind = NPY.GENBOOLLTR + char = NPY.BOOLLTR BoxType = boxes.W_BoolBox format_code = "?" @@ -551,31 +555,49 @@ class Int8(BaseType, Integer): T = rffi.SIGNEDCHAR + num = NPY.BYTE + kind = NPY.SIGNEDLTR + char = NPY.BYTELTR BoxType = boxes.W_Int8Box format_code = "b" class UInt8(BaseType, Integer): T = rffi.UCHAR + num = NPY.UBYTE + kind = NPY.UNSIGNEDLTR + char = NPY.UBYTELTR BoxType = boxes.W_UInt8Box format_code = "B" class Int16(BaseType, Integer): T = rffi.SHORT + num = NPY.SHORT + kind = NPY.SIGNEDLTR + char = NPY.SHORTLTR BoxType = boxes.W_Int16Box format_code = "h" class UInt16(BaseType, Integer): T = rffi.USHORT + num = NPY.USHORT + kind = NPY.UNSIGNEDLTR + char = NPY.USHORTLTR BoxType = boxes.W_UInt16Box format_code = "H" class Int32(BaseType, Integer): T = rffi.INT + num = NPY.INT + kind = NPY.SIGNEDLTR + char = NPY.INTLTR BoxType = boxes.W_Int32Box format_code = "i" class UInt32(BaseType, Integer): T = rffi.UINT + num = NPY.UINT + kind = NPY.UNSIGNEDLTR + char = NPY.UINTLTR BoxType = boxes.W_UInt32Box format_code = "I" @@ -594,6 +616,9 @@ class Int64(BaseType, Integer): T = rffi.LONGLONG + num = NPY.LONGLONG + kind = NPY.SIGNEDLTR + char = NPY.LONGLONGLTR BoxType = boxes.W_Int64Box format_code = "q" @@ -615,6 +640,9 @@ class UInt64(BaseType, Integer): T = rffi.ULONGLONG + num = NPY.ULONGLONG + kind = NPY.UNSIGNEDLTR + char = NPY.ULONGLONGLTR BoxType = boxes.W_UInt64Box format_code = "Q" @@ -622,6 +650,9 @@ class Long(BaseType, Integer): T = rffi.LONG + num = NPY.LONG + kind = NPY.SIGNEDLTR + char = NPY.LONGLTR BoxType = boxes.W_LongBox format_code = "l" @@ -640,6 +671,9 @@ class ULong(BaseType, Integer): T = rffi.ULONG + num = NPY.ULONG + kind = NPY.UNSIGNEDLTR + char = NPY.ULONGLTR BoxType = boxes.W_ULongBox format_code = "L" @@ -974,6 +1008,9 @@ class Float16(BaseType, Float): _STORAGE_T = rffi.USHORT T = rffi.SHORT + num = NPY.HALF + kind = NPY.FLOATINGLTR + char = NPY.HALFLTR BoxType = boxes.W_Float16Box @specialize.argtype(1) @@ -1014,11 +1051,17 @@ class Float32(BaseType, Float): T = rffi.FLOAT + num = NPY.FLOAT + kind = NPY.FLOATINGLTR + char = NPY.FLOATLTR BoxType = boxes.W_Float32Box format_code = "f" class Float64(BaseType, Float): T = rffi.DOUBLE + num = NPY.DOUBLE + kind = NPY.FLOATINGLTR + char = NPY.DOUBLELTR BoxType = boxes.W_Float64Box format_code = "d" @@ -1592,28 +1635,43 @@ class Complex64(ComplexFloating, BaseType): T = rffi.FLOAT + num = NPY.CFLOAT + kind = NPY.COMPLEXLTR + char = NPY.CFLOATLTR BoxType = boxes.W_Complex64Box ComponentBoxType = boxes.W_Float32Box class Complex128(ComplexFloating, BaseType): T = rffi.DOUBLE + num = NPY.CDOUBLE + kind = NPY.COMPLEXLTR + char = NPY.CDOUBLELTR BoxType = boxes.W_Complex128Box ComponentBoxType = boxes.W_Float64Box if boxes.long_double_size == 8: class FloatLong(BaseType, Float): T = rffi.DOUBLE + num = NPY.LONGDOUBLE + kind = NPY.FLOATINGLTR + char = NPY.LONGDOUBLELTR BoxType = boxes.W_FloatLongBox format_code = "d" class ComplexLong(ComplexFloating, BaseType): T = rffi.DOUBLE + num = NPY.CLONGDOUBLE + kind = NPY.COMPLEXLTR + char = NPY.CLONGDOUBLELTR BoxType = boxes.W_ComplexLongBox ComponentBoxType = boxes.W_FloatLongBox elif boxes.long_double_size in (12, 16): class FloatLong(BaseType, Float): T = rffi.LONGDOUBLE + num = NPY.LONGDOUBLE + kind = NPY.FLOATINGLTR + char = NPY.LONGDOUBLELTR BoxType = boxes.W_FloatLongBox def runpack_str(self, space, s): @@ -1631,6 +1689,9 @@ class ComplexLong(ComplexFloating, BaseType): T = rffi.LONGDOUBLE + num = NPY.CLONGDOUBLE + kind = NPY.COMPLEXLTR + char = NPY.CLONGDOUBLELTR BoxType = boxes.W_ComplexLongBox ComponentBoxType = boxes.W_FloatLongBox @@ -1638,6 +1699,9 @@ class ObjectType(Primitive, BaseType): T = lltype.Signed + num = NPY.OBJECT + kind = NPY.OBJECTLTR + char = NPY.OBJECTLTR BoxType = boxes.W_ObjectBox def get_element_size(self): @@ -1698,7 +1762,7 @@ else: raise oefmt(self.space.w_NotImplementedError, "object dtype cannot unbox %s", str(box)) - + @specialize.argtype(1) def box(self, w_obj): if isinstance(w_obj, W_Root): @@ -1949,6 +2013,9 @@ class StringType(FlexibleType): T = lltype.Char + num = NPY.STRING + kind = NPY.STRINGLTR + char = NPY.STRINGLTR @jit.unroll_safe def coerce(self, space, dtype, w_item): @@ -2046,6 +2113,9 @@ class UnicodeType(FlexibleType): T = lltype.Char + num = NPY.UNICODE + kind = NPY.UNICODELTR + char = NPY.UNICODELTR def get_element_size(self): return 4 # always UTF-32 @@ -2110,6 +2180,9 @@ class VoidType(FlexibleType): T = lltype.Char + num = NPY.VOID + kind = NPY.VOIDLTR + char = NPY.VOIDLTR def _coerce(self, space, arr, ofs, dtype, w_items, shape): # TODO: Make sure the shape and the array match @@ -2196,6 +2269,9 @@ class RecordType(FlexibleType): T = lltype.Char + num = NPY.VOID + kind = NPY.VOIDLTR + char = NPY.VOIDLTR def read(self, arr, i, offset, dtype=None): if dtype is None: From noreply at buildbot.pypy.org Fri Apr 24 07:00:47 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Fri, 24 Apr 2015 07:00:47 +0200 (CEST) Subject: [pypy-commit] pypy can_cast: remove num, kind, char from W_Dtype Message-ID: <20150424050047.D2AEC1C069E@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: can_cast Changeset: r76914:05d75330bba0 Date: 2015-04-24 06:00 +0100 http://bitbucket.org/pypy/pypy/changeset/05d75330bba0/ Log: remove num, kind, char from W_Dtype diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py --- a/pypy/module/micronumpy/descriptor.py +++ b/pypy/module/micronumpy/descriptor.py @@ -42,18 +42,12 @@ class W_Dtype(W_Root): _immutable_fields_ = [ - "itemtype?", "num", "kind", "char", "w_box_type", - "byteorder?", "names?", "fields?", "elsize?", "alignment?", - "shape?", "subdtype?", "base?", - ] + "itemtype?", "w_box_type", "byteorder?", "names?", "fields?", + "elsize?", "alignment?", "shape?", "subdtype?", "base?"] - def __init__(self, itemtype, num, kind, char, w_box_type, - byteorder=None, names=[], fields={}, - elsize=None, shape=[], subdtype=None): + def __init__(self, itemtype, w_box_type, byteorder=None, names=[], + fields={}, elsize=None, shape=[], subdtype=None): self.itemtype = itemtype - self.num = num - self.kind = kind - self.char = char self.w_box_type = w_box_type if byteorder is None: if itemtype.get_element_size() == 1 or isinstance(itemtype, types.ObjectType): @@ -74,6 +68,18 @@ else: self.base = subdtype.base + @property + def num(self): + return self.itemtype.num + + @property + def kind(self): + return self.itemtype.kind + + @property + def char(self): + return self.itemtype.char + def __repr__(self): if self.fields: return '' % self.fields @@ -453,7 +459,7 @@ fields = self.fields if fields is None: fields = {} - return W_Dtype(itemtype, self.num, self.kind, self.char, + return W_Dtype(itemtype, self.w_box_type, byteorder=endian, elsize=self.elsize, names=self.names, fields=fields, shape=self.shape, subdtype=self.subdtype) @@ -488,8 +494,7 @@ fields[fldname] = (offset, subdtype) offset += subdtype.elsize names.append(fldname) - return W_Dtype(types.RecordType(space), NPY.VOID, NPY.VOIDLTR, NPY.VOIDLTR, - space.gettypefor(boxes.W_VoidBox), + return W_Dtype(types.RecordType(space), space.gettypefor(boxes.W_VoidBox), names=names, fields=fields, elsize=offset) @@ -556,7 +561,7 @@ if size == 1: return subdtype size *= subdtype.elsize - return W_Dtype(types.VoidType(space), NPY.VOID, NPY.VOIDLTR, NPY.VOIDLTR, + return W_Dtype(types.VoidType(space), space.gettypefor(boxes.W_VoidBox), shape=shape, subdtype=subdtype, elsize=size) @@ -657,7 +662,10 @@ except ValueError: raise oefmt(space.w_TypeError, "data type not understood") if char == NPY.CHARLTR: - return new_string_dtype(space, 1, NPY.CHARLTR) + return W_Dtype( + types.CharType(space), + elsize=1, + w_box_type=space.gettypefor(boxes.W_StringBox)) elif char == NPY.STRINGLTR or char == NPY.STRINGLTR2: return new_string_dtype(space, size) elif char == NPY.UNICODELTR: @@ -667,13 +675,10 @@ assert False -def new_string_dtype(space, size, char=NPY.STRINGLTR): +def new_string_dtype(space, size): return W_Dtype( types.StringType(space), elsize=size, - num=NPY.STRING, - kind=NPY.STRINGLTR, - char=char, w_box_type=space.gettypefor(boxes.W_StringBox), ) @@ -683,9 +688,6 @@ return W_Dtype( itemtype, elsize=size * itemtype.get_element_size(), - num=NPY.UNICODE, - kind=NPY.UNICODELTR, - char=NPY.UNICODELTR, w_box_type=space.gettypefor(boxes.W_UnicodeBox), ) @@ -694,9 +696,6 @@ return W_Dtype( types.VoidType(space), elsize=size, - num=NPY.VOID, - kind=NPY.VOIDLTR, - char=NPY.VOIDLTR, w_box_type=space.gettypefor(boxes.W_VoidBox), ) @@ -705,159 +704,93 @@ def __init__(self, space): self.w_booldtype = W_Dtype( types.Bool(space), - num=NPY.BOOL, - kind=NPY.GENBOOLLTR, - char=NPY.BOOLLTR, w_box_type=space.gettypefor(boxes.W_BoolBox), ) self.w_int8dtype = W_Dtype( types.Int8(space), - num=NPY.BYTE, - kind=NPY.SIGNEDLTR, - char=NPY.BYTELTR, w_box_type=space.gettypefor(boxes.W_Int8Box), ) self.w_uint8dtype = W_Dtype( types.UInt8(space), - num=NPY.UBYTE, - kind=NPY.UNSIGNEDLTR, - char=NPY.UBYTELTR, w_box_type=space.gettypefor(boxes.W_UInt8Box), ) self.w_int16dtype = W_Dtype( types.Int16(space), - num=NPY.SHORT, - kind=NPY.SIGNEDLTR, - char=NPY.SHORTLTR, w_box_type=space.gettypefor(boxes.W_Int16Box), ) self.w_uint16dtype = W_Dtype( types.UInt16(space), - num=NPY.USHORT, - kind=NPY.UNSIGNEDLTR, - char=NPY.USHORTLTR, w_box_type=space.gettypefor(boxes.W_UInt16Box), ) self.w_int32dtype = W_Dtype( types.Int32(space), - num=NPY.INT, - kind=NPY.SIGNEDLTR, - char=NPY.INTLTR, w_box_type=space.gettypefor(boxes.W_Int32Box), ) self.w_uint32dtype = W_Dtype( types.UInt32(space), - num=NPY.UINT, - kind=NPY.UNSIGNEDLTR, - char=NPY.UINTLTR, w_box_type=space.gettypefor(boxes.W_UInt32Box), ) self.w_longdtype = W_Dtype( types.Long(space), - num=NPY.LONG, - kind=NPY.SIGNEDLTR, - char=NPY.LONGLTR, w_box_type=space.gettypefor(boxes.W_LongBox), ) self.w_ulongdtype = W_Dtype( types.ULong(space), - num=NPY.ULONG, - kind=NPY.UNSIGNEDLTR, - char=NPY.ULONGLTR, w_box_type=space.gettypefor(boxes.W_ULongBox), ) self.w_int64dtype = W_Dtype( types.Int64(space), - num=NPY.LONGLONG, - kind=NPY.SIGNEDLTR, - char=NPY.LONGLONGLTR, w_box_type=space.gettypefor(boxes.W_Int64Box), ) self.w_uint64dtype = W_Dtype( types.UInt64(space), - num=NPY.ULONGLONG, - kind=NPY.UNSIGNEDLTR, - char=NPY.ULONGLONGLTR, w_box_type=space.gettypefor(boxes.W_UInt64Box), ) self.w_float32dtype = W_Dtype( types.Float32(space), - num=NPY.FLOAT, - kind=NPY.FLOATINGLTR, - char=NPY.FLOATLTR, w_box_type=space.gettypefor(boxes.W_Float32Box), ) self.w_float64dtype = W_Dtype( types.Float64(space), - num=NPY.DOUBLE, - kind=NPY.FLOATINGLTR, - char=NPY.DOUBLELTR, w_box_type=space.gettypefor(boxes.W_Float64Box), ) self.w_floatlongdtype = W_Dtype( types.FloatLong(space), - num=NPY.LONGDOUBLE, - kind=NPY.FLOATINGLTR, - char=NPY.LONGDOUBLELTR, w_box_type=space.gettypefor(boxes.W_FloatLongBox), ) self.w_complex64dtype = W_Dtype( types.Complex64(space), - num=NPY.CFLOAT, - kind=NPY.COMPLEXLTR, - char=NPY.CFLOATLTR, w_box_type=space.gettypefor(boxes.W_Complex64Box), ) self.w_complex128dtype = W_Dtype( types.Complex128(space), - num=NPY.CDOUBLE, - kind=NPY.COMPLEXLTR, - char=NPY.CDOUBLELTR, w_box_type=space.gettypefor(boxes.W_Complex128Box), ) self.w_complexlongdtype = W_Dtype( types.ComplexLong(space), - num=NPY.CLONGDOUBLE, - kind=NPY.COMPLEXLTR, - char=NPY.CLONGDOUBLELTR, w_box_type=space.gettypefor(boxes.W_ComplexLongBox), ) self.w_stringdtype = W_Dtype( types.StringType(space), elsize=0, - num=NPY.STRING, - kind=NPY.STRINGLTR, - char=NPY.STRINGLTR, w_box_type=space.gettypefor(boxes.W_StringBox), ) self.w_unicodedtype = W_Dtype( types.UnicodeType(space), elsize=0, - num=NPY.UNICODE, - kind=NPY.UNICODELTR, - char=NPY.UNICODELTR, w_box_type=space.gettypefor(boxes.W_UnicodeBox), ) self.w_voiddtype = W_Dtype( types.VoidType(space), elsize=0, - num=NPY.VOID, - kind=NPY.VOIDLTR, - char=NPY.VOIDLTR, w_box_type=space.gettypefor(boxes.W_VoidBox), ) self.w_float16dtype = W_Dtype( types.Float16(space), - num=NPY.HALF, - kind=NPY.FLOATINGLTR, - char=NPY.HALFLTR, w_box_type=space.gettypefor(boxes.W_Float16Box), ) self.w_objectdtype = W_Dtype( types.ObjectType(space), - num=NPY.OBJECT, - kind=NPY.OBJECTLTR, - char=NPY.OBJECTLTR, w_box_type=space.gettypefor(boxes.W_ObjectBox), ) aliases = { diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -2267,6 +2267,9 @@ "item() for Void aray with no fields not implemented")) return space.newtuple(ret_unwrapped) +class CharType(StringType): + char = NPY.CHARLTR + class RecordType(FlexibleType): T = lltype.Char num = NPY.VOID From noreply at buildbot.pypy.org Fri Apr 24 07:13:47 2015 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 24 Apr 2015 07:13:47 +0200 (CEST) Subject: [pypy-commit] buildbot default: add anubis64 Message-ID: <20150424051347.59AEE1C069E@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r949:98c105c0aaef Date: 2015-04-24 08:14 +0300 http://bitbucket.org/pypy/buildbot/changeset/98c105c0aaef/ Log: add anubis64 diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -36,3 +36,4 @@ *-win-x86-64 slave/pypy-buildbot master/pypy-buildbot +*.swp diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -439,7 +439,7 @@ # 'category' : 'mac64', # }, {"name": WIN32, - "slavenames": ["SalsaSalsa", "allegro_win32"], + "slavenames": ["SalsaSalsa", "allegro_win32", "anubis64"], "builddir": WIN32, "factory": pypyOwnTestFactoryWin, "locks": [WinSlaveLock.access('counting')], @@ -453,7 +453,7 @@ "category": "win32", }, {"name" : JITWIN32, - "slavenames": ["SalsaSalsa", "allegro_win32"], + "slavenames": ["SalsaSalsa", "allegro_win32", "anubis64"], 'builddir' : JITWIN32, 'factory' : pypyJITTranslatedTestFactoryWin, "locks": [WinSlaveLock.access('counting')], From noreply at buildbot.pypy.org Fri Apr 24 07:52:31 2015 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 24 Apr 2015 07:52:31 +0200 (CEST) Subject: [pypy-commit] pypy default: skip if translated, test is useful for development only Message-ID: <20150424055231.68F2D1C069E@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76915:8117b0f94004 Date: 2015-04-24 08:42 +0300 http://bitbucket.org/pypy/pypy/changeset/8117b0f94004/ Log: skip if translated, test is useful for development only diff --git a/pypy/module/micronumpy/test/test_object_arrays.py b/pypy/module/micronumpy/test/test_object_arrays.py --- a/pypy/module/micronumpy/test/test_object_arrays.py +++ b/pypy/module/micronumpy/test/test_object_arrays.py @@ -1,7 +1,12 @@ from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest +from pypy.conftest import option class AppTestObjectDtypes(BaseNumpyAppTest): + def setup_class(cls): + BaseNumpyAppTest.setup_class.im_func(cls) + cls.w_runappdirect = cls.space.wrap(option.runappdirect) + def test_scalar_from_object(self): from numpy import array import sys @@ -109,6 +114,8 @@ def test_array_interface(self): import numpy as np + if self.runappdirect: + skip('requires numpy.core, test with numpy test suite instead') import sys class DummyArray(object): def __init__(self, interface, base=None): From noreply at buildbot.pypy.org Fri Apr 24 10:12:41 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 24 Apr 2015 10:12:41 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: fixes Message-ID: <20150424081241.8AA5E1C022E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1782:e69cf8f75f84 Date: 2015-04-23 09:08 +0200 http://bitbucket.org/cffi/cffi/changeset/e69cf8f75f84/ Log: fixes diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -4086,13 +4086,11 @@ boffsetmax = (boffsetmax + 7) / 8; /* bits -> bytes */ boffsetmax = (boffsetmax + alignment - 1) & ~(alignment-1); if (totalsize < 0) { - totalsize = boffsetmax; - if (totalsize == 0) - totalsize = 1; + totalsize = boffsetmax ? boffsetmax : 1; } else { - if (detect_custom_layout(ct, sflags, boffsetmax, totalsize, - "wrong total size", "", "") < 0) + if (detect_custom_layout(ct, sflags, boffsetmax ? boffsetmax : 1, + totalsize, "wrong total size", "", "") < 0) goto error; if (totalsize < boffsetmax) { PyErr_Format(PyExc_TypeError, diff --git a/cffi/api.py b/cffi/api.py --- a/cffi/api.py +++ b/cffi/api.py @@ -570,7 +570,9 @@ def _set_cdef_types(ffi): struct_unions = [] pending_completion = [] - for name, tp in sorted(ffi._parser._declarations.items()): + lst = ffi._parser._declarations.items() + lst = sorted(lst, key=lambda x: x[0].split(' ', 1)[1]) + for name, tp in lst: kind, basename = name.split(' ', 1) if kind == 'struct' or kind == 'union': if kind == 'struct': diff --git a/new/test_dlopen.py b/new/test_dlopen.py --- a/new/test_dlopen.py +++ b/new/test_dlopen.py @@ -13,6 +13,12 @@ ffi.cdef("union foo_s { int a, b; };") assert ffi.sizeof("union foo_s") == 4 +def test_cdef_struct_union(): + ffi = FFI() + ffi.cdef("union bar_s { int a; }; struct foo_s { int b; };") + assert ffi.sizeof("union bar_s") == 4 + assert ffi.sizeof("struct foo_s") == 4 + def test_math_sin(): py.test.skip("XXX redo!") ffi = FFI() From noreply at buildbot.pypy.org Fri Apr 24 10:12:42 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 24 Apr 2015 10:12:42 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: in-progress Message-ID: <20150424081242.89AB81C022E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1783:68760daf07ab Date: 2015-04-24 09:00 +0200 http://bitbucket.org/cffi/cffi/changeset/68760daf07ab/ Log: in-progress diff --git a/cffi/api.py b/cffi/api.py --- a/cffi/api.py +++ b/cffi/api.py @@ -568,23 +568,48 @@ return ffi._get_cached_btype(tp) def _set_cdef_types(ffi): + from . import model + + all_structs = {} + all_typedefs = {} + for name, tp in ffi._parser._declarations.items(): + kind, basename = name.split(' ', 1) + if kind == 'struct' or kind == 'union' or kind == 'anonymous': + all_structs[tp.name] = tp + elif kind == 'typedef': + all_typedefs[basename] = tp + if getattr(tp, "origin", None) == "unknown_type": + all_structs[tp.name] = tp + elif isinstance(tp, model.NamedPointerType): + all_structs[tp.totype.name] = tp.totype + struct_unions = [] pending_completion = [] - lst = ffi._parser._declarations.items() - lst = sorted(lst, key=lambda x: x[0].split(' ', 1)[1]) - for name, tp in lst: - kind, basename = name.split(' ', 1) - if kind == 'struct' or kind == 'union': - if kind == 'struct': - BType = _cffi1_backend.new_struct_type(basename) - else: - BType = _cffi1_backend.new_union_type(basename) - struct_unions.append(basename) - struct_unions.append(BType) - if not tp.partial and tp.fldtypes is not None: - pending_completion.append((tp, BType)) + for name, tp in sorted(all_structs.items()): + if not isinstance(tp, model.UnionType): + BType = _cffi1_backend.new_struct_type(name) + else: + BType = _cffi1_backend.new_union_type(name) + struct_unions.append(name) + struct_unions.append(BType) + if not tp.partial and tp.fldtypes is not None: + pending_completion.append((tp, BType)) # - ffi.__set_types(struct_unions) + typenames = [] + for name, tp in sorted(all_typedefs.items()): + cname = tp._get_c_name() + if cname == name: + assert isinstance(tp, model.StructOrUnionOrEnum) + cname = '%s %s' % (tp.kind, tp.name) + try: + BType = ffi.typeof(cname) + except ffi.error: + ffi.__set_types(struct_unions, typenames) + BType = ffi.typeof(cname) + typenames.append(name) + typenames.append(BType) + # + ffi.__set_types(struct_unions, typenames) # for tp, BType in pending_completion: fldtypes = [ffi.typeof(ftp._get_c_name()) for ftp in tp.fldtypes] diff --git a/cffi/model.py b/cffi/model.py --- a/cffi/model.py +++ b/cffi/model.py @@ -459,7 +459,7 @@ def unknown_ptr_type(name, structname=None): if structname is None: - structname = '*$%s' % name + structname = '$$%s' % name tp = StructType(structname, None, None, None) return NamedPointerType(tp, name) diff --git a/new/ffi_obj.c b/new/ffi_obj.c --- a/new/ffi_obj.c +++ b/new/ffi_obj.c @@ -525,15 +525,18 @@ static PyObject *ffi__set_types(FFIObject *self, PyObject *args) { - PyObject *lst1; + PyObject *lst1, *lst2; _cffi_opcode_t *types = NULL; struct _cffi_struct_union_s *struct_unions = NULL; + struct _cffi_typename_s *typenames = NULL; - if (!PyArg_ParseTuple(args, "O!", &PyList_Type, &lst1)) + if (!PyArg_ParseTuple(args, "O!O!", + &PyList_Type, &lst1, &PyList_Type, &lst2)) return NULL; if (self->ctx_is_static) { bad_usage: + PyMem_Free(typenames); PyMem_Free(struct_unions); PyMem_Free(types); if (!PyErr_Occurred()) @@ -543,19 +546,24 @@ cleanup_builder_c(self->types_builder); - int i, lst_length = PyList_GET_SIZE(lst1) / 2; - Py_ssize_t new_size_1 = sizeof(_cffi_opcode_t) * lst_length; - Py_ssize_t new_size_2 = sizeof(struct _cffi_struct_union_s) * lst_length; - types = PyMem_Malloc(new_size_1); - struct_unions = PyMem_Malloc(new_size_2); - if (!types || !struct_unions) { + int i; + int lst1_length = PyList_GET_SIZE(lst1) / 2; + int lst2_length = PyList_GET_SIZE(lst2) / 2; + Py_ssize_t newsize0 = sizeof(_cffi_opcode_t) * (lst1_length + lst2_length); + Py_ssize_t newsize1 = sizeof(struct _cffi_struct_union_s) * lst1_length; + Py_ssize_t newsize2 = sizeof(struct _cffi_typename_s) * lst2_length; + types = PyMem_Malloc(newsize0); + struct_unions = PyMem_Malloc(newsize1); + typenames = PyMem_Malloc(newsize2); + if (!types || !struct_unions || !typenames) { PyErr_NoMemory(); goto bad_usage; } - memset(types, 0, new_size_1); - memset(struct_unions, 0, new_size_2); + memset(types, 0, newsize0); + memset(struct_unions, 0, newsize1); + memset(typenames, 0, newsize2); - for (i = 0; i < lst_length; i++) { + for (i = 0; i < lst1_length; i++) { PyObject *x = PyList_GET_ITEM(lst1, i * 2); if (!PyString_Check(x)) goto bad_usage; @@ -570,18 +578,32 @@ struct_unions[i].size = (size_t)-2; struct_unions[i].alignment = -2; } - for (i = 0; i < lst_length; i++) { + for (i = 0; i < lst2_length; i++) { + PyObject *x = PyList_GET_ITEM(lst2, i * 2); + if (!PyString_Check(x)) + goto bad_usage; + typenames[i].name = PyString_AS_STRING(x); + typenames[i].type_index = lst1_length + i; + + x = PyList_GET_ITEM(lst2, i * 2 + 1); + if (!CTypeDescr_Check(x)) + goto bad_usage; + types[lst1_length + i] = x; + } + for (i = 0; i < lst1_length + lst2_length; i++) { PyObject *x = (PyObject *)types[i]; Py_INCREF(x); } - Py_INCREF(lst1); /* to keep alive the strings in '.name' */ + Py_INCREF(args); /* to keep alive the strings in '.name' */ Py_XDECREF(self->dynamic_types); - self->dynamic_types = lst1; + self->dynamic_types = args; self->types_builder->ctx.types = types; - self->types_builder->num_types_imported = lst_length; + self->types_builder->num_types_imported = lst1_length + lst2_length; self->types_builder->ctx.struct_unions = struct_unions; - self->types_builder->ctx.num_struct_unions = lst_length; + self->types_builder->ctx.num_struct_unions = lst1_length; + self->types_builder->ctx.typenames = typenames; + self->types_builder->ctx.num_typenames = lst2_length; Py_INCREF(Py_None); return Py_None; diff --git a/new/parse_c_type.c b/new/parse_c_type.c --- a/new/parse_c_type.c +++ b/new/parse_c_type.c @@ -56,7 +56,9 @@ static int is_ident_first(char x) { - return (('A' <= x && x <= 'Z') || ('a' <= x && x <= 'z') || x == '_'); + return (('A' <= x && x <= 'Z') || ('a' <= x && x <= 'z') || x == '_' || + x == '$'); /* '$' in names is supported here, for the struct + names invented by cparser */ } static int is_digit(char x) diff --git a/new/recompiler.py b/new/recompiler.py --- a/new/recompiler.py +++ b/new/recompiler.py @@ -508,8 +508,17 @@ _generate_cpy_union_collecttype = _generate_cpy_struct_collecttype def _generate_cpy_struct_decl(self, tp, name): - cname = tp.get_c_name('') - self._struct_decl(tp, cname, cname.replace(' ', '_')) + cname = tp._get_c_name() + if ' ' in cname: + prefix, declname = cname.split(' ', 1) + else: + prefix, declname = '', cname + while declname.startswith('$'): + prefix += 'D' + declname = declname[1:] + approxname = prefix + '_' + declname + assert '$' not in approxname + self._struct_decl(tp, cname, approxname) _generate_cpy_union_decl = _generate_cpy_struct_decl def _generate_cpy_struct_ctx(self, tp, name, prefix='s'): @@ -518,28 +527,6 @@ _generate_cpy_union_ctx = _generate_cpy_struct_ctx # ---------- - # 'anonymous' declarations. These are produced for anonymous structs - # or unions; the 'name' is obtained by a typedef. - - def _generate_cpy_anonymous_collecttype(self, tp, name): - if isinstance(tp, model.EnumType): - self._generate_cpy_enum_collecttype(tp, name) - else: - self._struct_collecttype(tp) - - def _generate_cpy_anonymous_decl(self, tp, name): - if isinstance(tp, model.EnumType): - self._generate_cpy_enum_decl(tp, name, '') - else: - self._struct_decl(tp, name, 'typedef_' + name) - - def _generate_cpy_anonymous_ctx(self, tp, name): - if isinstance(tp, model.EnumType): - self._generate_cpy_enum_ctx(tp, name, '') - else: - self._struct_ctx(tp, name, 'typedef_' + name) - - # ---------- # constants, declared with "static const ..." def _generate_cpy_const(self, is_int, name, tp=None, category='const', diff --git a/new/test_dlopen.py b/new/test_dlopen.py --- a/new/test_dlopen.py +++ b/new/test_dlopen.py @@ -19,6 +19,34 @@ assert ffi.sizeof("union bar_s") == 4 assert ffi.sizeof("struct foo_s") == 4 +def test_cdef_struct_typename_1(): + ffi = FFI() + ffi.cdef("typedef struct { int a; } t1; typedef struct { t1* m; } t2;") + assert ffi.sizeof("t2") == ffi.sizeof("void *") + assert ffi.sizeof("t1") == 4 + +def test_cdef_struct_typename_2(): + ffi = FFI() + ffi.cdef("typedef struct { int a; } *p1; typedef struct { p1 m; } *p2;") + p2 = ffi.new("p2") + assert ffi.sizeof(p2[0]) == ffi.sizeof("void *") + assert ffi.sizeof(p2[0].m) == ffi.sizeof("void *") + +def test_cdef_struct_anon_1(): + ffi = FFI() + ffi.cdef("typedef struct { int a; } t1; struct foo_s { t1* m; };") + assert ffi.sizeof("struct foo_s") == ffi.sizeof("void *") + +def test_cdef_struct_anon_2(): + ffi = FFI() + ffi.cdef("typedef struct { int a; } *p1; struct foo_s { p1 m; };") + assert ffi.sizeof("struct foo_s") == ffi.sizeof("void *") + +def test_cdef_struct_anon_3(): + ffi = FFI() + ffi.cdef("typedef struct { int a; } **pp; struct foo_s { pp m; };") + assert ffi.sizeof("struct foo_s") == ffi.sizeof("void *") + def test_math_sin(): py.test.skip("XXX redo!") ffi = FFI() From noreply at buildbot.pypy.org Fri Apr 24 10:12:43 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 24 Apr 2015 10:12:43 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Change of scope for now: the initial goal is now to avoid breaking Message-ID: <20150424081243.CFCBF1C022E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1784:6d8fa9486905 Date: 2015-04-24 09:17 +0200 http://bitbucket.org/cffi/cffi/changeset/6d8fa9486905/ Log: Change of scope for now: the initial goal is now to avoid breaking ffi.dlopen() and ffi.verify(), and only add _cffi1.recompile(). diff --git a/new/PLAN b/_cffi1/PLAN rename from new/PLAN rename to _cffi1/PLAN diff --git a/new/_cffi_include.h b/_cffi1/_cffi_include.h rename from new/_cffi_include.h rename to _cffi1/_cffi_include.h diff --git a/new/bsdopendirtype.py b/_cffi1/bsdopendirtype.py rename from new/bsdopendirtype.py rename to _cffi1/bsdopendirtype.py diff --git a/new/bsdopendirtype_build.py b/_cffi1/bsdopendirtype_build.py rename from new/bsdopendirtype_build.py rename to _cffi1/bsdopendirtype_build.py diff --git a/new/cffi-1.0.rst b/_cffi1/cffi-1.0.rst rename from new/cffi-1.0.rst rename to _cffi1/cffi-1.0.rst diff --git a/new/cffi1 b/_cffi1/cffi1 rename from new/cffi1 rename to _cffi1/cffi1 diff --git a/new/cffi1_module.c b/_cffi1/cffi1_module.c rename from new/cffi1_module.c rename to _cffi1/cffi1_module.c diff --git a/new/cffi_opcode.py b/_cffi1/cffi_opcode.py rename from new/cffi_opcode.py rename to _cffi1/cffi_opcode.py diff --git a/new/cglob.c b/_cffi1/cglob.c rename from new/cglob.c rename to _cffi1/cglob.c diff --git a/new/ffi_obj.c b/_cffi1/ffi_obj.c rename from new/ffi_obj.c rename to _cffi1/ffi_obj.c diff --git a/new/lib_obj.c b/_cffi1/lib_obj.c rename from new/lib_obj.c rename to _cffi1/lib_obj.c diff --git a/new/manual.c b/_cffi1/manual.c rename from new/manual.c rename to _cffi1/manual.c diff --git a/new/parse_c_type.c b/_cffi1/parse_c_type.c rename from new/parse_c_type.c rename to _cffi1/parse_c_type.c diff --git a/new/parse_c_type.h b/_cffi1/parse_c_type.h rename from new/parse_c_type.h rename to _cffi1/parse_c_type.h diff --git a/new/readdir2.py b/_cffi1/readdir2.py rename from new/readdir2.py rename to _cffi1/readdir2.py diff --git a/new/readdir2_build.py b/_cffi1/readdir2_build.py rename from new/readdir2_build.py rename to _cffi1/readdir2_build.py diff --git a/new/realize_c_type.c b/_cffi1/realize_c_type.c rename from new/realize_c_type.c rename to _cffi1/realize_c_type.c diff --git a/new/recompiler.py b/_cffi1/recompiler.py rename from new/recompiler.py rename to _cffi1/recompiler.py diff --git a/new/setup.py b/_cffi1/setup.py rename from new/setup.py rename to _cffi1/setup.py diff --git a/new/setup_manual.py b/_cffi1/setup_manual.py rename from new/setup_manual.py rename to _cffi1/setup_manual.py diff --git a/new/test_dlopen.py b/_cffi1/test_dlopen.py rename from new/test_dlopen.py rename to _cffi1/test_dlopen.py diff --git a/new/test_ffi_obj.py b/_cffi1/test_ffi_obj.py rename from new/test_ffi_obj.py rename to _cffi1/test_ffi_obj.py diff --git a/new/test_parse_c_type.py b/_cffi1/test_parse_c_type.py rename from new/test_parse_c_type.py rename to _cffi1/test_parse_c_type.py diff --git a/new/test_realize_c_type.py b/_cffi1/test_realize_c_type.py rename from new/test_realize_c_type.py rename to _cffi1/test_realize_c_type.py diff --git a/new/test_recompiler.py b/_cffi1/test_recompiler.py rename from new/test_recompiler.py rename to _cffi1/test_recompiler.py diff --git a/new/test_verify1.py b/_cffi1/test_verify1.py rename from new/test_verify1.py rename to _cffi1/test_verify1.py diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -5402,6 +5402,12 @@ return NULL; } +static PyObject *b__get_types(PyObject *self, PyObject *noarg) +{ + return PyTuple_Pack(2, (PyObject *)&CData_Type, + (PyObject *)&CTypeDescr_Type); +} + /************************************************************/ static char _testfunc0(char a, char b) @@ -5706,6 +5712,7 @@ #ifdef MS_WIN32 {"getwinerror", b_getwinerror, METH_VARARGS}, #endif + {"_get_types", b__get_types, METH_NOARGS}, {"_testfunc", b__testfunc, METH_VARARGS}, {"_testbuff", b__testbuff, METH_VARARGS}, {NULL, NULL} /* Sentinel */ @@ -5820,7 +5827,7 @@ } #endif -#include "../new/cffi1_module.c" +#include "../_cffi1/cffi1_module.c" static void *cffi_exports[] = { (void *)26, @@ -5861,7 +5868,7 @@ #if PY_MAJOR_VERSION >= 3 static struct PyModuleDef FFIBackendModuleDef = { PyModuleDef_HEAD_INIT, - "_cffi1_backend", + "_cffi_backend", NULL, -1, FFIBackendMethods, @@ -5875,7 +5882,7 @@ #define INITERROR return PyMODINIT_FUNC -init_cffi1_backend(void) +init_cffi_backend(void) #endif { PyObject *m, *v; @@ -5892,7 +5899,7 @@ #if PY_MAJOR_VERSION >= 3 m = PyModule_Create(&FFIBackendModuleDef); #else - m = Py_InitModule("_cffi1_backend", FFIBackendMethods); + m = Py_InitModule("_cffi_backend", FFIBackendMethods); #endif if (m == NULL) diff --git a/cffi/__init__.py b/cffi/__init__.py --- a/cffi/__init__.py +++ b/cffi/__init__.py @@ -1,10 +1,13 @@ __all__ = ['FFI', 'VerificationError', 'VerificationMissing', 'CDefError', 'FFIError'] -from .api import FFI, CDefError +from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -FFIError = FFI.error # backward compatibility - __version__ = "1.0.0" __version_info__ = (1, 0, 0) + +# The verifier module file names are based on the CRC32 of a string that +# contains the following version number. It may be older than __version__ +# if nothing is clearly incompatible. +__version_verifier_modules__ = "0.8.6" diff --git a/cffi/api.py b/cffi/api.py --- a/cffi/api.py +++ b/cffi/api.py @@ -1,9 +1,5 @@ import sys, types from .lock import allocate_lock -import _cffi1_backend - -class DeprecatedError(Exception): - pass try: callable @@ -19,7 +15,8 @@ basestring = str -FFIError = _cffi1_backend.FFI.error +class FFIError(Exception): + pass class CDefError(Exception): def __str__(self): @@ -30,7 +27,7 @@ return '%s%s' % (line, self.args[0]) -class FFI(_cffi1_backend.FFI): +class FFI(object): r''' The main top-level class that you instantiate once, or once per module. @@ -48,16 +45,21 @@ C.printf("hello, %s!\n", ffi.new("char[]", "world")) ''' - def __init__(self): + def __init__(self, backend=None): """Create an FFI instance. The 'backend' argument is used to select a non-default backend, mostly for tests. """ from . import cparser, model - from . import __version__ - - backend = _cffi1_backend - assert backend.__version__ == __version__, \ - "version mismatch, %s != %s" % (backend.__version__, __version__) + if backend is None: + # You need PyPy (>= 2.0 beta), or a CPython (>= 2.6) with + # _cffi_backend.so compiled. + import _cffi_backend as backend + from . import __version__ + assert backend.__version__ == __version__, \ + "version mismatch, %s != %s" % (backend.__version__, __version__) + # (If you insist you can also try to pass the option + # 'backend=backend_ctypes.CTypesBackend()', but don't + # rely on it! It's probably not going to work well.) self._backend = backend self._lock = allocate_lock() @@ -75,9 +77,18 @@ if name.startswith('RTLD_'): setattr(self, name, getattr(backend, name)) # - #with self._lock: - # self.BVoidP = self._get_cached_btype(model.voidp_type) - # self.BCharA = self._get_cached_btype(model.char_array_type) + with self._lock: + self.BVoidP = self._get_cached_btype(model.voidp_type) + self.BCharA = self._get_cached_btype(model.char_array_type) + if isinstance(backend, types.ModuleType): + # _cffi_backend: attach these constants to the class + if not hasattr(FFI, 'NULL'): + FFI.NULL = self.cast(self.BVoidP, 0) + FFI.CData, FFI.CType = backend._get_types() + else: + # ctypes backend: attach these constants to the instance + self.NULL = self.cast(self.BVoidP, 0) + self.CData, self.CType = backend._get_types() def cdef(self, csource, override=False, packed=False): """Parse the given C source. This registers all declared functions, @@ -97,7 +108,6 @@ if override: for cache in self._function_caches: cache.clear() - _set_cdef_types(self) def dlopen(self, name, flags=0): """Load and return a dynamic library identified by 'name'. @@ -145,7 +155,7 @@ "pointer-to-function type" % (cdecl,)) return btype - def XXXtypeof(self, cdecl): + def typeof(self, cdecl): """Parse the C type given as a string and return the corresponding object. It can also be used on 'cdata' instance to get its C type. @@ -164,7 +174,7 @@ return self._get_cached_btype(cdecl._cffi_base_type) raise TypeError(type(cdecl)) - def XXXsizeof(self, cdecl): + def sizeof(self, cdecl): """Return the size in bytes of the argument. It can be a string naming a C type, or a 'cdata' instance. """ @@ -174,7 +184,7 @@ else: return self._backend.sizeof(cdecl) - def XXXalignof(self, cdecl): + def alignof(self, cdecl): """Return the natural alignment size in bytes of the C type given as a string. """ @@ -182,7 +192,7 @@ cdecl = self._typeof(cdecl) return self._backend.alignof(cdecl) - def XXXoffsetof(self, cdecl, *fields_or_indexes): + def offsetof(self, cdecl, *fields_or_indexes): """Return the offset of the named field inside the given structure or array, which must be given as a C type name. You can give several field names in case of nested structures. @@ -193,7 +203,7 @@ cdecl = self._typeof(cdecl) return self._typeoffsetof(cdecl, *fields_or_indexes)[1] - def XXXnew(self, cdecl, init=None): + def new(self, cdecl, init=None): """Allocate an instance according to the specified C type and return a pointer to it. The specified C type must be either a pointer or an array: ``new('X *')`` allocates an X and returns @@ -220,7 +230,7 @@ cdecl = self._typeof(cdecl) return self._backend.newp(cdecl, init) - def XXXcast(self, cdecl, source): + def cast(self, cdecl, source): """Similar to a C cast: returns an instance of the named C type initialized with the given 'source'. The source is casted between integers or pointers of any type. @@ -229,7 +239,7 @@ cdecl = self._typeof(cdecl) return self._backend.cast(cdecl, source) - def XXXstring(self, cdata, maxlen=-1): + def string(self, cdata, maxlen=-1): """Return a Python string (or unicode string) from the 'cdata'. If 'cdata' is a pointer or array of characters or bytes, returns the null-terminated string. The returned string extends until @@ -247,7 +257,7 @@ """ return self._backend.string(cdata, maxlen) - def XXXbuffer(self, cdata, size=-1): + def buffer(self, cdata, size=-1): """Return a read-write buffer object that references the raw C data pointed to by the given 'cdata'. The 'cdata' must be a pointer or an array. Can be passed to functions expecting a buffer, or directly @@ -260,7 +270,7 @@ """ return self._backend.buffer(cdata, size) - def XXXfrom_buffer(self, python_buffer): + def from_buffer(self, python_buffer): """Return a that points to the data of the given Python object, which must support the buffer interface. Note that this is not meant to be used on the built-in types str, @@ -270,7 +280,7 @@ """ return self._backend.from_buffer(self.BCharA, python_buffer) - def XXXcallback(self, cdecl, python_callable=None, error=None): + def callback(self, cdecl, python_callable=None, error=None): """Return a callback object or a decorator making such a callback object. 'cdecl' must name a C function pointer type. The callback invokes the specified 'python_callable' (which may @@ -290,7 +300,7 @@ else: return callback_decorator_wrap(python_callable) # direct mode - def XXXgetctype(self, cdecl, replace_with=''): + def getctype(self, cdecl, replace_with=''): """Return a string giving the C type 'cdecl', which may be itself a string or a object. If 'replace_with' is given, it gives extra text to append (or insert for more complicated C types), like @@ -306,7 +316,7 @@ replace_with = ' ' + replace_with return self._backend.getcname(cdecl, replace_with) - def XXXgc(self, cdata, destructor): + def gc(self, cdata, destructor): """Return a new cdata object that points to the same data. Later, when this new cdata object is garbage-collected, 'destructor(old_cdata_object)' will be called. @@ -320,15 +330,18 @@ return gc_weakrefs.build(cdata, destructor) def _get_cached_btype(self, type): - raise DeprecatedError + assert self._lock.acquire(False) is False + # call me with the lock! + try: + BType = self._cached_btypes[type] + except KeyError: + finishlist = [] + BType = type.get_cached_btype(self, finishlist) + for type in finishlist: + type.finish_backend_type(self, finishlist) + return BType - def verify(self, source='', **kwargs): - from recompiler import verify # XXX must be in the current dir - FFI._verify_counter += 1 - return verify(self, 'verify%d' % FFI._verify_counter, source, **kwargs) - _verify_counter = 0 - - def XXXverify(self, source='', tmpdir=None, **kwargs): + def verify(self, source='', tmpdir=None, **kwargs): """Verify that the current ffi signatures compile on this machine, and return a dynamic library object. The dynamic library can be used to call functions and access global @@ -362,10 +375,10 @@ return self._backend.get_errno() def _set_errno(self, errno): self._backend.set_errno(errno) - XXXerrno = property(_get_errno, _set_errno, None, + errno = property(_get_errno, _set_errno, None, "the value of 'errno' from/to the C calls") - def XXXgetwinerror(self, code=-1): + def getwinerror(self, code=-1): return self._backend.getwinerror(code) def _pointer_to(self, ctype): @@ -373,7 +386,7 @@ with self._lock: return model.pointer_cache(self, ctype) - def XXXaddressof(self, cdata, *fields_or_indexes): + def addressof(self, cdata, *fields_or_indexes): """Return the address of a . If 'fields_or_indexes' are given, returns the address of that field or array item in the structure or array, recursively in @@ -405,7 +418,6 @@ variables, which must anyway be accessed directly from the lib object returned by the original FFI instance. """ - XXX with ffi_to_include._lock: with self._lock: self._parser.include(ffi_to_include._parser) @@ -413,10 +425,10 @@ self._cdefsources.extend(ffi_to_include._cdefsources) self._cdefsources.append(']') - def XXXnew_handle(self, x): + def new_handle(self, x): return self._backend.newp_handle(self.BVoidP, x) - def XXXfrom_handle(self, x): + def from_handle(self, x): return self._backend.from_handle(x) def set_unicode(self, enabled_flag): @@ -426,7 +438,6 @@ declare these types to be (pointers to) plain 8-bit characters. This is mostly for backward compatibility; you usually want True. """ - XXX if self._windows_unicode is not None: raise ValueError("set_unicode() can only be called once") enabled_flag = bool(enabled_flag) @@ -566,56 +577,3 @@ else: with ffi._lock: return ffi._get_cached_btype(tp) - -def _set_cdef_types(ffi): - from . import model - - all_structs = {} - all_typedefs = {} - for name, tp in ffi._parser._declarations.items(): - kind, basename = name.split(' ', 1) - if kind == 'struct' or kind == 'union' or kind == 'anonymous': - all_structs[tp.name] = tp - elif kind == 'typedef': - all_typedefs[basename] = tp - if getattr(tp, "origin", None) == "unknown_type": - all_structs[tp.name] = tp - elif isinstance(tp, model.NamedPointerType): - all_structs[tp.totype.name] = tp.totype - - struct_unions = [] - pending_completion = [] - for name, tp in sorted(all_structs.items()): - if not isinstance(tp, model.UnionType): - BType = _cffi1_backend.new_struct_type(name) - else: - BType = _cffi1_backend.new_union_type(name) - struct_unions.append(name) - struct_unions.append(BType) - if not tp.partial and tp.fldtypes is not None: - pending_completion.append((tp, BType)) - # - typenames = [] - for name, tp in sorted(all_typedefs.items()): - cname = tp._get_c_name() - if cname == name: - assert isinstance(tp, model.StructOrUnionOrEnum) - cname = '%s %s' % (tp.kind, tp.name) - try: - BType = ffi.typeof(cname) - except ffi.error: - ffi.__set_types(struct_unions, typenames) - BType = ffi.typeof(cname) - typenames.append(name) - typenames.append(BType) - # - ffi.__set_types(struct_unions, typenames) - # - for tp, BType in pending_completion: - fldtypes = [ffi.typeof(ftp._get_c_name()) for ftp in tp.fldtypes] - lst = list(zip(tp.fldnames, fldtypes, tp.fldbitsize)) - sflags = 0 - if tp.packed: - sflags = 8 # SF_PACKED - _cffi1_backend.complete_struct_or_union(BType, lst, ffi, - -1, -1, sflags) diff --git a/setup_base.py b/setup_base.py --- a/setup_base.py +++ b/setup_base.py @@ -9,7 +9,9 @@ from distutils.core import setup from distutils.extension import Extension standard = '__pypy__' not in sys.modules - setup(ext_modules=[Extension(name = '_cffi1_backend', + setup(packages=['cffi'], + requires=['pycparser'], + ext_modules=[Extension(name = '_cffi_backend', include_dirs=include_dirs, sources=sources, libraries=libraries, From noreply at buildbot.pypy.org Fri Apr 24 10:12:44 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 24 Apr 2015 10:12:44 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Fix for test_verify Message-ID: <20150424081244.DEDF71C022E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1785:548efd04854c Date: 2015-04-24 09:23 +0200 http://bitbucket.org/cffi/cffi/changeset/548efd04854c/ Log: Fix for test_verify diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -3802,7 +3802,7 @@ CTypeDescrObject *ct; PyObject *fields, *interned_fields, *ignored; int is_union, alignment; - Py_ssize_t boffset, i, nb_fields, boffsetmax; + Py_ssize_t boffset, i, nb_fields, boffsetmax, alignedsize; Py_ssize_t totalsize = -1; int totalalignment = -1; CFieldObject **previous; @@ -4084,12 +4084,15 @@ as 1 instead. But for ctypes support, we allow the manually- specified totalsize to be zero in this case. */ boffsetmax = (boffsetmax + 7) / 8; /* bits -> bytes */ - boffsetmax = (boffsetmax + alignment - 1) & ~(alignment-1); + alignedsize = (boffsetmax + alignment - 1) & ~(alignment-1); + if (alignedsize == 0) + alignedsize = 1; + if (totalsize < 0) { - totalsize = boffsetmax ? boffsetmax : 1; + totalsize = alignedsize; } else { - if (detect_custom_layout(ct, sflags, boffsetmax ? boffsetmax : 1, + if (detect_custom_layout(ct, sflags, alignedsize, totalsize, "wrong total size", "", "") < 0) goto error; if (totalsize < boffsetmax) { From noreply at buildbot.pypy.org Fri Apr 24 10:12:45 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 24 Apr 2015 10:12:45 +0200 (CEST) Subject: [pypy-commit] cffi default: Skip an existing test after figuring out that it is indeed likely to Message-ID: <20150424081245.EB27F1C022E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1786:a6987485dc7f Date: 2015-04-24 10:13 +0200 http://bitbucket.org/cffi/cffi/changeset/a6987485dc7f/ Log: Skip an existing test after figuring out that it is indeed likely to crash in test_vgen, depending on details of the platform. Changed it to clearly flag this as invalid already in the cdef, and why. diff --git a/cffi/model.py b/cffi/model.py --- a/cffi/model.py +++ b/cffi/model.py @@ -140,6 +140,16 @@ replace_with = self._base_pattern % (', '.join(reprargs),) self.c_name_with_marker = ( self.result.c_name_with_marker.replace('&', replace_with)) + # + if isinstance(result, StructOrUnion) and result.partial: + from .ffiplatform import VerificationError + raise VerificationError( + '%s: the %s is a struct with "...;", which is not ' + 'supported as return type (how to call it with ' + 'libffi depends on possibly-omitted fields). ' + 'Workaround: write a wrapper function which takes ' + 'a pointer-to-struct as extra argument and writes ' + 'the result there' % (self, result)) class RawFunctionType(BaseFunctionType): diff --git a/testing/test_verify.py b/testing/test_verify.py --- a/testing/test_verify.py +++ b/testing/test_verify.py @@ -1233,11 +1233,13 @@ py.test.skip('Segfaults on mips64el') # XXX bad abuse of "struct { ...; }". It only works a bit by chance # anyway. XXX think about something better :-( + # ...in fact, it is no longer supported: likely crashes in vgen ffi = FFI() - ffi.cdef(""" + py.test.raises(VerificationError, ffi.cdef, """ typedef struct { ...; } myhandle_t; myhandle_t foo(void); """) + py.test.skip("XXX reimplement maybe?") lib = ffi.verify(""" typedef short myhandle_t; myhandle_t foo(void) { return 42; } @@ -1245,6 +1247,21 @@ h = lib.foo() assert ffi.sizeof(h) == ffi.sizeof("short") +def test_return_partial_struct(): + py.test.skip("not implemented") + ffi = FFI() + ffi.cdef(""" + typedef struct { int x; ...; } foo_t; + foo_t foo(void); + """) + lib = ffi.verify(""" + typedef struct { int y, x; } foo_t; + foo_t foo(void) { foo_t r = { 45, 81 }; return r; } + """) + h = lib.foo() + assert ffi.sizeof(h) == 2 * ffi.sizeof("int") + assert h.x == 81 + def test_cannot_name_struct_type(): ffi = FFI() ffi.cdef("typedef struct { int x; } *sp; void foo(sp);") From noreply at buildbot.pypy.org Fri Apr 24 10:13:48 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 24 Apr 2015 10:13:48 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: hg merge default Message-ID: <20150424081348.3F5C71C022E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1787:f58471f990dd Date: 2015-04-24 10:13 +0200 http://bitbucket.org/cffi/cffi/changeset/f58471f990dd/ Log: hg merge default diff --git a/cffi/model.py b/cffi/model.py --- a/cffi/model.py +++ b/cffi/model.py @@ -140,6 +140,16 @@ replace_with = self._base_pattern % (', '.join(reprargs),) self.c_name_with_marker = ( self.result.c_name_with_marker.replace('&', replace_with)) + # + if isinstance(result, StructOrUnion) and result.partial: + from .ffiplatform import VerificationError + raise VerificationError( + '%s: the %s is a struct with "...;", which is not ' + 'supported as return type (how to call it with ' + 'libffi depends on possibly-omitted fields). ' + 'Workaround: write a wrapper function which takes ' + 'a pointer-to-struct as extra argument and writes ' + 'the result there' % (self, result)) class RawFunctionType(BaseFunctionType): diff --git a/testing/test_verify.py b/testing/test_verify.py --- a/testing/test_verify.py +++ b/testing/test_verify.py @@ -657,9 +657,9 @@ # case the 'static' is completely ignored. ffi.cdef("static const int AA, BB, CC, DD;") lib = ffi.verify("#define AA 42\n" - "#define BB (-43)\n" - "#define CC (22*2)\n" - "#define DD ((unsigned int)142)\n") + "#define BB (-43) // blah\n" + "#define CC (22*2) /* foobar */\n" + "#define DD ((unsigned int)142) /* foo\nbar */\n") assert lib.AA == 42 assert lib.BB == -43 assert lib.CC == 44 @@ -1233,11 +1233,13 @@ py.test.skip('Segfaults on mips64el') # XXX bad abuse of "struct { ...; }". It only works a bit by chance # anyway. XXX think about something better :-( + # ...in fact, it is no longer supported: likely crashes in vgen ffi = FFI() - ffi.cdef(""" + py.test.raises(VerificationError, ffi.cdef, """ typedef struct { ...; } myhandle_t; myhandle_t foo(void); """) + py.test.skip("XXX reimplement maybe?") lib = ffi.verify(""" typedef short myhandle_t; myhandle_t foo(void) { return 42; } @@ -1245,6 +1247,21 @@ h = lib.foo() assert ffi.sizeof(h) == ffi.sizeof("short") +def test_return_partial_struct(): + py.test.skip("not implemented") + ffi = FFI() + ffi.cdef(""" + typedef struct { int x; ...; } foo_t; + foo_t foo(void); + """) + lib = ffi.verify(""" + typedef struct { int y, x; } foo_t; + foo_t foo(void) { foo_t r = { 45, 81 }; return r; } + """) + h = lib.foo() + assert ffi.sizeof(h) == 2 * ffi.sizeof("int") + assert h.x == 81 + def test_cannot_name_struct_type(): ffi = FFI() ffi.cdef("typedef struct { int x; } *sp; void foo(sp);") From noreply at buildbot.pypy.org Fri Apr 24 10:33:37 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 24 Apr 2015 10:33:37 +0200 (CEST) Subject: [pypy-commit] pypy vmprof2: Remove usage of "uintptr_t" and use "unsigned long", which is more like Message-ID: <20150424083337.96A121C061E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof2 Changeset: r76916:79023f7a715e Date: 2015-04-24 10:33 +0200 http://bitbucket.org/pypy/pypy/changeset/79023f7a715e/ Log: Remove usage of "uintptr_t" and use "unsigned long", which is more like the rest of the code. Avoid calling __sync_lock_test_and_set on Windows and directly call the "proper" functions. diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py --- a/rpython/jit/backend/llsupport/codemap.py +++ b/rpython/jit/backend/llsupport/codemap.py @@ -26,29 +26,19 @@ srcdir = os.path.join(os.path.dirname(__file__), 'src') eci = ExternalCompilationInfo(post_include_bits=[""" -#if defined _MSC_VER && _MSC_VER < 1600 -#ifdef _WIN32 -typedef unsigned int uintptr_t; -#else -typedef usigned long uintptr_t; -#endif -#else -#include -#endif - -RPY_EXTERN long pypy_jit_codemap_add(uintptr_t addr, +RPY_EXTERN long pypy_jit_codemap_add(unsigned long addr, unsigned int machine_code_size, long *bytecode_info, unsigned int bytecode_info_size); -RPY_EXTERN long *pypy_jit_codemap_del(uintptr_t addr, unsigned int size); -RPY_EXTERN uintptr_t pypy_jit_codemap_firstkey(void); +RPY_EXTERN long *pypy_jit_codemap_del(unsigned long addr, unsigned int size); +RPY_EXTERN unsigned long pypy_jit_codemap_firstkey(void); RPY_EXTERN void *pypy_find_codemap_at_addr(long addr, long* start_addr); RPY_EXTERN long pypy_yield_codemap_at_addr(void *codemap_raw, long addr, long *current_pos_addr); -RPY_EXTERN long pypy_jit_depthmap_add(uintptr_t addr, unsigned int size, +RPY_EXTERN long pypy_jit_depthmap_add(unsigned long addr, unsigned int size, unsigned int stackdepth); -RPY_EXTERN void pypy_jit_depthmap_clear(uintptr_t addr, unsigned int size); +RPY_EXTERN void pypy_jit_depthmap_clear(unsigned long addr, unsigned int size); """], separate_module_sources=[ open(os.path.join(srcdir, 'skiplist.c'), 'r').read() + diff --git a/rpython/jit/backend/llsupport/src/codemap.c b/rpython/jit/backend/llsupport/src/codemap.c --- a/rpython/jit/backend/llsupport/src/codemap.c +++ b/rpython/jit/backend/llsupport/src/codemap.c @@ -9,10 +9,15 @@ void pypy_codemap_invalid_set(int value) { +#ifndef _MSC_VER if (value) __sync_lock_test_and_set(&pypy_codemap_currently_invalid, 1); else __sync_lock_release(&pypy_codemap_currently_invalid); +#else + InterlockedExchange((long volatile *)&pypy_codemap_currently_invalid, + (long)value); +#endif } @@ -31,7 +36,7 @@ /*** interface used from codemap.py ***/ RPY_EXTERN -long pypy_jit_codemap_add(uintptr_t addr, unsigned int machine_code_size, +long pypy_jit_codemap_add(unsigned long addr, unsigned int machine_code_size, long *bytecode_info, unsigned int bytecode_info_size) { skipnode_t *new = skiplist_malloc(sizeof(codemap_data_t)); @@ -52,9 +57,9 @@ } RPY_EXTERN -long *pypy_jit_codemap_del(uintptr_t addr, unsigned int size) +long *pypy_jit_codemap_del(unsigned long addr, unsigned int size) { - uintptr_t search_key = addr + size - 1; + unsigned long search_key = addr + size - 1; long *result; skipnode_t *node; @@ -78,7 +83,7 @@ } RPY_EXTERN -uintptr_t pypy_jit_codemap_firstkey(void) +unsigned long pypy_jit_codemap_firstkey(void) { return skiplist_firstkey(&jit_codemap_head); } @@ -90,7 +95,7 @@ { skipnode_t *codemap = skiplist_search(&jit_codemap_head, addr); codemap_data_t *data; - uintptr_t rel_addr; + unsigned long rel_addr; if (codemap == &jit_codemap_head) { if (start_addr) @@ -98,7 +103,7 @@ return NULL; } - rel_addr = (uintptr_t)addr - codemap->key; + rel_addr = (unsigned long)addr - codemap->key; data = (codemap_data_t *)codemap->data; if (rel_addr >= data->machine_code_size) { if (start_addr) @@ -153,7 +158,7 @@ /*** interface used from codemap.py ***/ RPY_EXTERN -long pypy_jit_depthmap_add(uintptr_t addr, unsigned int size, +long pypy_jit_depthmap_add(unsigned long addr, unsigned int size, unsigned int stackdepth) { skipnode_t *new = skiplist_malloc(sizeof(depthmap_data_t)); @@ -173,9 +178,9 @@ } RPY_EXTERN -void pypy_jit_depthmap_clear(uintptr_t addr, unsigned int size) +void pypy_jit_depthmap_clear(unsigned long addr, unsigned int size) { - uintptr_t search_key = addr + size - 1; + unsigned long search_key = addr + size - 1; if (size == 0) return; @@ -196,14 +201,15 @@ RPY_EXTERN long pypy_jit_stack_depth_at_loc(long loc) { - skipnode_t *depthmap = skiplist_search(&jit_depthmap_head, (uintptr_t)loc); + skipnode_t *depthmap = skiplist_search(&jit_depthmap_head, + (unsigned long)loc); depthmap_data_t *data; - uintptr_t rel_addr; + unsigned long rel_addr; if (depthmap == &jit_depthmap_head) return -1; - rel_addr = (uintptr_t)loc - depthmap->key; + rel_addr = (unsigned long)loc - depthmap->key; data = (depthmap_data_t *)depthmap->data; if (rel_addr >= data->block_size) return -1; diff --git a/rpython/jit/backend/llsupport/src/skiplist.c b/rpython/jit/backend/llsupport/src/skiplist.c --- a/rpython/jit/backend/llsupport/src/skiplist.c +++ b/rpython/jit/backend/llsupport/src/skiplist.c @@ -1,40 +1,19 @@ #include -#if defined _MSC_VER - #include - int __sync_lock_test_and_set(int * i, int j) - { - return _interlockedbittestandreset(i, j); - } - int __sync_lock_release(int *i) - { - return _interlockedbittestandreset(i, 0); - } - #if _MSC_VER < 1600 - #ifdef _WIN32 - typedef unsigned int uintptr_t; - #else - typedef usigned long uintptr_t; - #endif - #endif -#else -#include -#endif - #define HAS_SKIPLIST #define SKIPLIST_HEIGHT 8 typedef struct skipnode_s { - uintptr_t key; + unsigned long key; char *data; struct skipnode_s *next[SKIPLIST_HEIGHT]; /* may be smaller */ } skipnode_t; -static skipnode_t *skiplist_malloc(uintptr_t datasize) +static skipnode_t *skiplist_malloc(unsigned long datasize) { char *result; - uintptr_t basesize; - uintptr_t length = 1; + unsigned long basesize; + unsigned long length = 1; while (length < SKIPLIST_HEIGHT && (rand() & 3) == 0) length++; basesize = sizeof(skipnode_t) - @@ -46,12 +25,12 @@ return (skipnode_t *)result; } -static skipnode_t *skiplist_search(skipnode_t *head, uintptr_t searchkey) +static skipnode_t *skiplist_search(skipnode_t *head, unsigned long searchkey) { /* Returns the skipnode with key closest (but <=) searchkey. Note that if there is no item with key <= searchkey in the list, this will return the head node. */ - uintptr_t level = SKIPLIST_HEIGHT - 1; + unsigned long level = SKIPLIST_HEIGHT - 1; while (1) { skipnode_t *next = head->next[level]; if (next != NULL && next->key <= searchkey) { @@ -68,13 +47,13 @@ static void skiplist_insert(skipnode_t *head, skipnode_t *new) { - uintptr_t size0 = sizeof(skipnode_t) - - SKIPLIST_HEIGHT * sizeof(skipnode_t *); - uintptr_t height_of_new = (new->data - ((char *)new + size0)) / - sizeof(skipnode_t *); + unsigned long size0 = sizeof(skipnode_t) - + SKIPLIST_HEIGHT * sizeof(skipnode_t *); + unsigned long height_of_new = (new->data - ((char *)new + size0)) / + sizeof(skipnode_t *); - uintptr_t level = SKIPLIST_HEIGHT - 1; - uintptr_t searchkey = new->key; + unsigned long level = SKIPLIST_HEIGHT - 1; + unsigned long searchkey = new->key; while (1) { skipnode_t *next = head->next[level]; if (next != NULL && next->key <= searchkey) { @@ -92,9 +71,9 @@ } } -static skipnode_t *skiplist_remove(skipnode_t *head, uintptr_t exact_key) +static skipnode_t *skiplist_remove(skipnode_t *head, unsigned long exact_key) { - uintptr_t level = SKIPLIST_HEIGHT - 1; + unsigned long level = SKIPLIST_HEIGHT - 1; while (1) { skipnode_t *next = head->next[level]; if (next != NULL && next->key <= exact_key) { @@ -115,7 +94,7 @@ } } -static uintptr_t skiplist_firstkey(skipnode_t *head) +static unsigned long skiplist_firstkey(skipnode_t *head) { if (head->next[0] == NULL) return 0; diff --git a/rpython/jit/backend/llsupport/test/test_skiplist.py b/rpython/jit/backend/llsupport/test/test_skiplist.py --- a/rpython/jit/backend/llsupport/test/test_skiplist.py +++ b/rpython/jit/backend/llsupport/test/test_skiplist.py @@ -5,16 +5,16 @@ ffi.cdef(""" typedef struct { - uintptr_t key; + unsigned long key; char *data; ...; } skipnode_t; -skipnode_t *skiplist_malloc(uintptr_t datasize); -skipnode_t *skiplist_search(skipnode_t *head, uintptr_t searchkey); +skipnode_t *skiplist_malloc(unsigned long datasize); +skipnode_t *skiplist_search(skipnode_t *head, unsigned long searchkey); void skiplist_insert(skipnode_t *head, skipnode_t *new); -skipnode_t *skiplist_remove(skipnode_t *head, uintptr_t exact_key); -uintptr_t skiplist_firstkey(skipnode_t *head); +skipnode_t *skiplist_remove(skipnode_t *head, unsigned long exact_key); +unsigned long skiplist_firstkey(skipnode_t *head); """) filename = os.path.join(os.path.dirname(__file__), '..', 'src', 'skiplist.c') From noreply at buildbot.pypy.org Fri Apr 24 11:06:38 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 24 Apr 2015 11:06:38 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: hg merge default Message-ID: <20150424090638.0C67F1C06F5@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1789:111b2f47845a Date: 2015-04-24 11:07 +0200 http://bitbucket.org/cffi/cffi/changeset/111b2f47845a/ Log: hg merge default diff --git a/cffi/model.py b/cffi/model.py --- a/cffi/model.py +++ b/cffi/model.py @@ -140,16 +140,6 @@ replace_with = self._base_pattern % (', '.join(reprargs),) self.c_name_with_marker = ( self.result.c_name_with_marker.replace('&', replace_with)) - # - if isinstance(result, StructOrUnion) and result.partial: - from .ffiplatform import VerificationError - raise VerificationError( - '%s: the %s is a struct with "...;", which is not ' - 'supported as return type (how to call it with ' - 'libffi depends on possibly-omitted fields). ' - 'Workaround: write a wrapper function which takes ' - 'a pointer-to-struct as extra argument and writes ' - 'the result there' % (self, result)) class RawFunctionType(BaseFunctionType): diff --git a/cffi/vengine_gen.py b/cffi/vengine_gen.py --- a/cffi/vengine_gen.py +++ b/cffi/vengine_gen.py @@ -149,15 +149,21 @@ context = 'argument of %s' % name arglist = [type.get_c_name(' %s' % arg, context) for type, arg in zip(tp.args, argnames)] + tpresult = tp.result + if isinstance(tpresult, model.StructOrUnion): + arglist.insert(0, tpresult.get_c_name(' *r', context)) + tpresult = model.void_type arglist = ', '.join(arglist) or 'void' wrappername = '_cffi_f_%s' % name self.export_symbols.append(wrappername) funcdecl = ' %s(%s)' % (wrappername, arglist) context = 'result of %s' % name - prnt(tp.result.get_c_name(funcdecl, context)) + prnt(tpresult.get_c_name(funcdecl, context)) prnt('{') # - if not isinstance(tp.result, model.VoidType): + if isinstance(tp.result, model.StructOrUnion): + result_code = '*r = ' + elif not isinstance(tp.result, model.VoidType): result_code = 'return ' else: result_code = '' @@ -174,15 +180,22 @@ else: indirections = [] base_tp = tp - if any(isinstance(typ, model.StructOrUnion) for typ in tp.args): + if (any(isinstance(typ, model.StructOrUnion) for typ in tp.args) + or isinstance(tp.result, model.StructOrUnion)): indirect_args = [] for i, typ in enumerate(tp.args): if isinstance(typ, model.StructOrUnion): typ = model.PointerType(typ) indirections.append((i, typ)) indirect_args.append(typ) + indirect_result = tp.result + if isinstance(indirect_result, model.StructOrUnion): + indirect_result = model.PointerType(indirect_result) + indirect_args.insert(0, indirect_result) + indirections.insert(0, ("result", indirect_result)) + indirect_result = model.void_type tp = model.FunctionPtrType(tuple(indirect_args), - tp.result, tp.ellipsis) + indirect_result, tp.ellipsis) BFunc = self.ffi._get_cached_btype(tp) wrappername = '_cffi_f_%s' % name newfunction = module.load_function(BFunc, wrappername) @@ -195,9 +208,16 @@ def _make_struct_wrapper(self, oldfunc, i, tp, base_tp): backend = self.ffi._backend BType = self.ffi._get_cached_btype(tp) - def newfunc(*args): - args = args[:i] + (backend.newp(BType, args[i]),) + args[i+1:] - return oldfunc(*args) + if i == "result": + ffi = self.ffi + def newfunc(*args): + res = ffi.new(BType) + oldfunc(res, *args) + return res[0] + else: + def newfunc(*args): + args = args[:i] + (backend.newp(BType, args[i]),) + args[i+1:] + return oldfunc(*args) newfunc._cffi_base_type = base_tp return newfunc diff --git a/testing/test_verify.py b/testing/test_verify.py --- a/testing/test_verify.py +++ b/testing/test_verify.py @@ -1233,13 +1233,11 @@ py.test.skip('Segfaults on mips64el') # XXX bad abuse of "struct { ...; }". It only works a bit by chance # anyway. XXX think about something better :-( - # ...in fact, it is no longer supported: likely crashes in vgen ffi = FFI() - py.test.raises(VerificationError, ffi.cdef, """ + ffi.cdef(""" typedef struct { ...; } myhandle_t; myhandle_t foo(void); """) - py.test.skip("XXX reimplement maybe?") lib = ffi.verify(""" typedef short myhandle_t; myhandle_t foo(void) { return 42; } @@ -1248,7 +1246,6 @@ assert ffi.sizeof(h) == ffi.sizeof("short") def test_return_partial_struct(): - py.test.skip("not implemented") ffi = FFI() ffi.cdef(""" typedef struct { int x; ...; } foo_t; @@ -1262,6 +1259,26 @@ assert ffi.sizeof(h) == 2 * ffi.sizeof("int") assert h.x == 81 +def test_take_and_return_partial_structs(): + ffi = FFI() + ffi.cdef(""" + typedef struct { int x; ...; } foo_t; + foo_t foo(foo_t, foo_t); + """) + lib = ffi.verify(""" + typedef struct { int y, x; } foo_t; + foo_t foo(foo_t a, foo_t b) { + foo_t r = { 100, a.x * 5 + b.x * 7 }; + return r; + } + """) + args = ffi.new("foo_t[3]") + args[0].x = 1000 + args[2].x = -498 + h = lib.foo(args[0], args[2]) + assert ffi.sizeof(h) == 2 * ffi.sizeof("int") + assert h.x == 1000 * 5 - 498 * 7 + def test_cannot_name_struct_type(): ffi = FFI() ffi.cdef("typedef struct { int x; } *sp; void foo(sp);") From noreply at buildbot.pypy.org Fri Apr 24 11:06:15 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 24 Apr 2015 11:06:15 +0200 (CEST) Subject: [pypy-commit] cffi default: Reimplement vengine_gen returning incomplete structs. Message-ID: <20150424090615.0267F1C06F5@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1788:e4b22b167187 Date: 2015-04-24 11:06 +0200 http://bitbucket.org/cffi/cffi/changeset/e4b22b167187/ Log: Reimplement vengine_gen returning incomplete structs. diff --git a/cffi/model.py b/cffi/model.py --- a/cffi/model.py +++ b/cffi/model.py @@ -140,16 +140,6 @@ replace_with = self._base_pattern % (', '.join(reprargs),) self.c_name_with_marker = ( self.result.c_name_with_marker.replace('&', replace_with)) - # - if isinstance(result, StructOrUnion) and result.partial: - from .ffiplatform import VerificationError - raise VerificationError( - '%s: the %s is a struct with "...;", which is not ' - 'supported as return type (how to call it with ' - 'libffi depends on possibly-omitted fields). ' - 'Workaround: write a wrapper function which takes ' - 'a pointer-to-struct as extra argument and writes ' - 'the result there' % (self, result)) class RawFunctionType(BaseFunctionType): diff --git a/cffi/vengine_gen.py b/cffi/vengine_gen.py --- a/cffi/vengine_gen.py +++ b/cffi/vengine_gen.py @@ -149,15 +149,21 @@ context = 'argument of %s' % name arglist = [type.get_c_name(' %s' % arg, context) for type, arg in zip(tp.args, argnames)] + tpresult = tp.result + if isinstance(tpresult, model.StructOrUnion): + arglist.insert(0, tpresult.get_c_name(' *r', context)) + tpresult = model.void_type arglist = ', '.join(arglist) or 'void' wrappername = '_cffi_f_%s' % name self.export_symbols.append(wrappername) funcdecl = ' %s(%s)' % (wrappername, arglist) context = 'result of %s' % name - prnt(tp.result.get_c_name(funcdecl, context)) + prnt(tpresult.get_c_name(funcdecl, context)) prnt('{') # - if not isinstance(tp.result, model.VoidType): + if isinstance(tp.result, model.StructOrUnion): + result_code = '*r = ' + elif not isinstance(tp.result, model.VoidType): result_code = 'return ' else: result_code = '' @@ -174,15 +180,22 @@ else: indirections = [] base_tp = tp - if any(isinstance(typ, model.StructOrUnion) for typ in tp.args): + if (any(isinstance(typ, model.StructOrUnion) for typ in tp.args) + or isinstance(tp.result, model.StructOrUnion)): indirect_args = [] for i, typ in enumerate(tp.args): if isinstance(typ, model.StructOrUnion): typ = model.PointerType(typ) indirections.append((i, typ)) indirect_args.append(typ) + indirect_result = tp.result + if isinstance(indirect_result, model.StructOrUnion): + indirect_result = model.PointerType(indirect_result) + indirect_args.insert(0, indirect_result) + indirections.insert(0, ("result", indirect_result)) + indirect_result = model.void_type tp = model.FunctionPtrType(tuple(indirect_args), - tp.result, tp.ellipsis) + indirect_result, tp.ellipsis) BFunc = self.ffi._get_cached_btype(tp) wrappername = '_cffi_f_%s' % name newfunction = module.load_function(BFunc, wrappername) @@ -195,9 +208,16 @@ def _make_struct_wrapper(self, oldfunc, i, tp, base_tp): backend = self.ffi._backend BType = self.ffi._get_cached_btype(tp) - def newfunc(*args): - args = args[:i] + (backend.newp(BType, args[i]),) + args[i+1:] - return oldfunc(*args) + if i == "result": + ffi = self.ffi + def newfunc(*args): + res = ffi.new(BType) + oldfunc(res, *args) + return res[0] + else: + def newfunc(*args): + args = args[:i] + (backend.newp(BType, args[i]),) + args[i+1:] + return oldfunc(*args) newfunc._cffi_base_type = base_tp return newfunc diff --git a/testing/test_verify.py b/testing/test_verify.py --- a/testing/test_verify.py +++ b/testing/test_verify.py @@ -1233,13 +1233,11 @@ py.test.skip('Segfaults on mips64el') # XXX bad abuse of "struct { ...; }". It only works a bit by chance # anyway. XXX think about something better :-( - # ...in fact, it is no longer supported: likely crashes in vgen ffi = FFI() - py.test.raises(VerificationError, ffi.cdef, """ + ffi.cdef(""" typedef struct { ...; } myhandle_t; myhandle_t foo(void); """) - py.test.skip("XXX reimplement maybe?") lib = ffi.verify(""" typedef short myhandle_t; myhandle_t foo(void) { return 42; } @@ -1248,7 +1246,6 @@ assert ffi.sizeof(h) == ffi.sizeof("short") def test_return_partial_struct(): - py.test.skip("not implemented") ffi = FFI() ffi.cdef(""" typedef struct { int x; ...; } foo_t; @@ -1262,6 +1259,26 @@ assert ffi.sizeof(h) == 2 * ffi.sizeof("int") assert h.x == 81 +def test_take_and_return_partial_structs(): + ffi = FFI() + ffi.cdef(""" + typedef struct { int x; ...; } foo_t; + foo_t foo(foo_t, foo_t); + """) + lib = ffi.verify(""" + typedef struct { int y, x; } foo_t; + foo_t foo(foo_t a, foo_t b) { + foo_t r = { 100, a.x * 5 + b.x * 7 }; + return r; + } + """) + args = ffi.new("foo_t[3]") + args[0].x = 1000 + args[2].x = -498 + h = lib.foo(args[0], args[2]) + assert ffi.sizeof(h) == 2 * ffi.sizeof("int") + assert h.x == 1000 * 5 - 498 * 7 + def test_cannot_name_struct_type(): ffi = FFI() ffi.cdef("typedef struct { int x; } *sp; void foo(sp);") From noreply at buildbot.pypy.org Fri Apr 24 11:20:37 2015 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 24 Apr 2015 11:20:37 +0200 (CEST) Subject: [pypy-commit] pypy default: Backed out changeset: d198d926afb8, more tests needed Message-ID: <20150424092037.98ED51C0726@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76917:8cb5c941efb4 Date: 2015-04-24 12:18 +0300 http://bitbucket.org/pypy/pypy/changeset/8cb5c941efb4/ Log: Backed out changeset: d198d926afb8, more tests needed diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py --- a/pypy/module/micronumpy/concrete.py +++ b/pypy/module/micronumpy/concrete.py @@ -328,11 +328,8 @@ return ArrayBuffer(self, readonly) def astype(self, space, dtype): - # we want to create a new array, but must respect the strides - # in self. So find a factor of the itemtype.elsize, and use this - factor = float(dtype.elsize) / self.dtype.elsize - strides = [int(factor*s) for s in self.get_strides()] - backstrides = [int(factor*s) for s in self.get_backstrides()] + strides, backstrides = calc_strides(self.get_shape(), dtype, + self.order) impl = ConcreteArray(self.get_shape(), dtype, self.order, strides, backstrides) loop.setslice(space, impl.get_shape(), impl, self) diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -2183,8 +2183,7 @@ assert b.dtype == 'bool' a = arange(6, dtype='f4').reshape(2,3) - b = a.T.astype('i4') - assert (a.T.strides == b.strides) + b = a.astype('i4') a = array('x').astype('S3').dtype assert a.itemsize == 3 From noreply at buildbot.pypy.org Fri Apr 24 11:53:42 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 24 Apr 2015 11:53:42 +0200 (CEST) Subject: [pypy-commit] cffi default: Fix for test_vgen.test_struct_returned_by_func (and uniformize the error message) Message-ID: <20150424095342.88EC91C061E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1790:3fcfce174377 Date: 2015-04-24 11:54 +0200 http://bitbucket.org/cffi/cffi/changeset/3fcfce174377/ Log: Fix for test_vgen.test_struct_returned_by_func (and uniformize the error message) diff --git a/cffi/vengine_gen.py b/cffi/vengine_gen.py --- a/cffi/vengine_gen.py +++ b/cffi/vengine_gen.py @@ -190,6 +190,10 @@ indirect_args.append(typ) indirect_result = tp.result if isinstance(indirect_result, model.StructOrUnion): + if indirect_result.fldtypes is None: + raise TypeError("'%s' is used as result type, " + "but is opaque" % ( + indirect_result._get_c_name(),)) indirect_result = model.PointerType(indirect_result) indirect_args.insert(0, indirect_result) indirections.insert(0, ("result", indirect_result)) diff --git a/testing/test_verify.py b/testing/test_verify.py --- a/testing/test_verify.py +++ b/testing/test_verify.py @@ -1680,9 +1680,8 @@ e = py.test.raises(TypeError, ffi.verify, "typedef struct { int x; } foo_t; " "foo_t myfunc(void) { foo_t x = { 42 }; return x; }") - assert str(e.value) in [ - "function myfunc: 'foo_t' is used as result type, but is opaque", - "function myfunc: result type 'foo_t' is opaque"] + assert str(e.value) == ( + "function myfunc: 'foo_t' is used as result type, but is opaque") def test_include(): ffi1 = FFI() From noreply at buildbot.pypy.org Fri Apr 24 11:53:47 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 24 Apr 2015 11:53:47 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: hg merge default Message-ID: <20150424095347.E7AEA1C061E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1791:d04c328c3d75 Date: 2015-04-24 11:54 +0200 http://bitbucket.org/cffi/cffi/changeset/d04c328c3d75/ Log: hg merge default diff --git a/cffi/vengine_gen.py b/cffi/vengine_gen.py --- a/cffi/vengine_gen.py +++ b/cffi/vengine_gen.py @@ -190,6 +190,10 @@ indirect_args.append(typ) indirect_result = tp.result if isinstance(indirect_result, model.StructOrUnion): + if indirect_result.fldtypes is None: + raise TypeError("'%s' is used as result type, " + "but is opaque" % ( + indirect_result._get_c_name(),)) indirect_result = model.PointerType(indirect_result) indirect_args.insert(0, indirect_result) indirections.insert(0, ("result", indirect_result)) diff --git a/testing/test_verify.py b/testing/test_verify.py --- a/testing/test_verify.py +++ b/testing/test_verify.py @@ -1680,9 +1680,8 @@ e = py.test.raises(TypeError, ffi.verify, "typedef struct { int x; } foo_t; " "foo_t myfunc(void) { foo_t x = { 42 }; return x; }") - assert str(e.value) in [ - "function myfunc: 'foo_t' is used as result type, but is opaque", - "function myfunc: result type 'foo_t' is opaque"] + assert str(e.value) == ( + "function myfunc: 'foo_t' is used as result type, but is opaque") def test_include(): ffi1 = FFI() From noreply at buildbot.pypy.org Fri Apr 24 11:55:26 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 24 Apr 2015 11:55:26 +0200 (CEST) Subject: [pypy-commit] cffi default: I *think* there is a chance for this to work now Message-ID: <20150424095526.E43A81C061E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1792:14c34ed8b40a Date: 2015-04-24 11:55 +0200 http://bitbucket.org/cffi/cffi/changeset/14c34ed8b40a/ Log: I *think* there is a chance for this to work now diff --git a/testing/test_verify.py b/testing/test_verify.py --- a/testing/test_verify.py +++ b/testing/test_verify.py @@ -1226,11 +1226,11 @@ xxx def test_opaque_integer_as_function_result(): - import platform - if platform.machine().startswith('sparc'): - py.test.skip('Breaks horribly on sparc (SIGILL + corrupted stack)') - elif platform.machine() == 'mips64' and sys.maxsize > 2**32: - py.test.skip('Segfaults on mips64el') + #import platform + #if platform.machine().startswith('sparc'): + # py.test.skip('Breaks horribly on sparc (SIGILL + corrupted stack)') + #elif platform.machine() == 'mips64' and sys.maxsize > 2**32: + # py.test.skip('Segfaults on mips64el') # XXX bad abuse of "struct { ...; }". It only works a bit by chance # anyway. XXX think about something better :-( ffi = FFI() From noreply at buildbot.pypy.org Fri Apr 24 12:10:03 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 24 Apr 2015 12:10:03 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Fix the tests so that they mostly run again Message-ID: <20150424101003.E50141C01D6@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1793:59c86e07b247 Date: 2015-04-24 12:10 +0200 http://bitbucket.org/cffi/cffi/changeset/59c86e07b247/ Log: Fix the tests so that they mostly run again diff --git a/_cffi1/__init__.py b/_cffi1/__init__.py new file mode 100644 --- /dev/null +++ b/_cffi1/__init__.py @@ -0,0 +1,1 @@ +from recompiler import make_c_source, recompile diff --git a/_cffi1/_cffi_include.h b/_cffi1/_cffi_include.h --- a/_cffi1/_cffi_include.h +++ b/_cffi1/_cffi_include.h @@ -167,7 +167,7 @@ PyObject *module, *c_api_object = NULL; void *src; - module = PyImport_ImportModule("_cffi1_backend"); + module = PyImport_ImportModule("_cffi_backend"); if (module == NULL) goto failure; @@ -181,7 +181,7 @@ src = PyCapsule_GetPointer(c_api_object, "cffi"); if ((uintptr_t)(((void **)src)[0]) < _CFFI_NUM_EXPORTS) { PyErr_SetString(PyExc_ImportError, - "the _cffi1_backend module is an outdated version"); + "the _cffi_backend module is an outdated version"); goto failure; } memcpy(_cffi_exports, src, _CFFI_NUM_EXPORTS * sizeof(void *)); diff --git a/_cffi1/cffi1 b/_cffi1/cffi1 deleted file mode 120000 --- a/_cffi1/cffi1 +++ /dev/null @@ -1,1 +0,0 @@ -../cffi \ No newline at end of file diff --git a/_cffi1/recompiler.py b/_cffi1/recompiler.py --- a/_cffi1/recompiler.py +++ b/_cffi1/recompiler.py @@ -1,5 +1,5 @@ import os, sys -from cffi1 import ffiplatform, model +from cffi import ffiplatform, model from cffi_opcode import * @@ -183,7 +183,7 @@ prnt('};') prnt() # - # the init function, loading _cffi1_backend and calling a method there + # the init function, loading _cffi_backend and calling a method there prnt('PyMODINIT_FUNC') prnt('init%s(void)' % (self.module_name,)) prnt('{') @@ -695,6 +695,6 @@ for name in dir(module.ffi): if not name.startswith('_'): attr = getattr(module.ffi, name) - if attr is not getattr(ffi, name): + if attr is not getattr(ffi, name, object()): setattr(ffi, name, attr) return module.lib diff --git a/_cffi1/test_dlopen.py b/_cffi1/test_dlopen.py --- a/_cffi1/test_dlopen.py +++ b/_cffi1/test_dlopen.py @@ -1,4 +1,6 @@ import py +py.test.skip("later") + from cffi1 import FFI import math @@ -48,7 +50,6 @@ assert ffi.sizeof("struct foo_s") == ffi.sizeof("void *") def test_math_sin(): - py.test.skip("XXX redo!") ffi = FFI() ffi.cdef("double sin(double);") m = ffi.dlopen('m') diff --git a/_cffi1/test_ffi_obj.py b/_cffi1/test_ffi_obj.py --- a/_cffi1/test_ffi_obj.py +++ b/_cffi1/test_ffi_obj.py @@ -1,5 +1,5 @@ import py -import _cffi1_backend +import _cffi_backend as _cffi1_backend def test_ffi_new(): diff --git a/_cffi1/test_parse_c_type.py b/_cffi1/test_parse_c_type.py --- a/_cffi1/test_parse_c_type.py +++ b/_cffi1/test_parse_c_type.py @@ -251,9 +251,9 @@ '->', Pointer(0)] def test_cffi_opcode_sync(): - import cffi_opcode, cffi1.model + import cffi_opcode, cffi.model for name in dir(lib): if name.startswith('_CFFI_'): assert getattr(cffi_opcode, name[6:]) == getattr(lib, name) assert sorted(cffi_opcode.PRIMITIVE_TO_INDEX.keys()) == ( - sorted(cffi1.model.PrimitiveType.ALL_PRIMITIVE_TYPES.keys())) + sorted(cffi.model.PrimitiveType.ALL_PRIMITIVE_TYPES.keys())) diff --git a/_cffi1/test_realize_c_type.py b/_cffi1/test_realize_c_type.py --- a/_cffi1/test_realize_c_type.py +++ b/_cffi1/test_realize_c_type.py @@ -2,8 +2,8 @@ def check(input, expected_output=None, expected_ffi_error=False): - import _cffi1_backend - ffi = _cffi1_backend.FFI() + import _cffi_backend + ffi = _cffi_backend.FFI() if not expected_ffi_error: ct = ffi.typeof(input) assert isinstance(ct, ffi.CType) diff --git a/_cffi1/test_recompiler.py b/_cffi1/test_recompiler.py --- a/_cffi1/test_recompiler.py +++ b/_cffi1/test_recompiler.py @@ -1,6 +1,6 @@ import py -import recompiler -from cffi1 import FFI +from cffi import FFI +from _cffi1 import recompiler def check_type_table(input, expected_output): diff --git a/_cffi1/test_verify1.py b/_cffi1/test_verify1.py --- a/_cffi1/test_verify1.py +++ b/_cffi1/test_verify1.py @@ -1,5 +1,6 @@ import sys, math, py -from cffi1 import FFI, VerificationError, model +from cffi import FFI, VerificationError, model +from _cffi1 import recompiler lib_m = ['m'] if sys.platform == 'win32': @@ -7,7 +8,7 @@ import distutils.ccompiler if distutils.ccompiler.get_default_compiler() == 'msvc': lib_m = ['msvcrt'] - pass # no obvious -Werror equivalent on MSVC + extra_compile_args = [] # no obvious -Werror equivalent on MSVC else: if (sys.platform == 'darwin' and [int(x) for x in os.uname()[2].split('.')] >= [11, 0, 0]): @@ -19,10 +20,13 @@ # assume a standard gcc extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion'] - class FFI(FFI): - def verify(self, *args, **kwds): - return super(FFI, self).verify( - *args, extra_compile_args=extra_compile_args, **kwds) +class FFI(FFI): + _verify_counter = 0 + def verify(self, preamble='', *args, **kwds): + FFI._verify_counter += 1 + return recompiler.verify(self, 'verify%d' % FFI._verify_counter, + preamble, *args, + extra_compile_args=extra_compile_args, **kwds) class U(object): def __add__(self, other): @@ -49,8 +53,8 @@ def test_missing_function_import_error(): # uses the original FFI that just gives a warning during compilation - import cffi1 - test_missing_function(ffi=cffi1.FFI()) + import cffi + test_missing_function(ffi=cffi.FFI()) def test_simple_case(): ffi = FFI() From noreply at buildbot.pypy.org Fri Apr 24 12:52:40 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 24 Apr 2015 12:52:40 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Fix these two demos Message-ID: <20150424105240.6B6921C03F7@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1794:fdb29c447984 Date: 2015-04-24 12:53 +0200 http://bitbucket.org/cffi/cffi/changeset/fdb29c447984/ Log: Fix these two demos diff --git a/_cffi1/bsdopendirtype_build.py b/_cffi1/bsdopendirtype_build.py --- a/_cffi1/bsdopendirtype_build.py +++ b/_cffi1/bsdopendirtype_build.py @@ -1,5 +1,5 @@ -from cffi1 import FFI -import recompiler +from cffi import FFI +from _cffi1 import recompile ffi = FFI() ffi.cdef(""" @@ -15,7 +15,7 @@ static const int DT_BLK, DT_CHR, DT_DIR, DT_FIFO, DT_LNK, DT_REG, DT_SOCK; """) -recompiler.recompile(ffi, "_bsdopendirtype", """ +recompile(ffi, "_bsdopendirtype", """ #include #include """) diff --git a/_cffi1/readdir2_build.py b/_cffi1/readdir2_build.py --- a/_cffi1/readdir2_build.py +++ b/_cffi1/readdir2_build.py @@ -1,5 +1,5 @@ -from cffi1 import FFI -from recompiler import recompile +from cffi import FFI +from _cffi1 import recompile ffi = FFI() ffi.cdef(""" diff --git a/_cffi1/recompiler.py b/_cffi1/recompiler.py --- a/_cffi1/recompiler.py +++ b/_cffi1/recompiler.py @@ -672,11 +672,7 @@ include_dirs.insert(0, '.') # XXX return ffiplatform.get_extension(source_name, module_name, **kwds) -def recompile(ffi, module_name, preamble, tmpdir=None, **kwds): - if tmpdir is None: - tmpdir = 'build' - if not os.path.isdir(tmpdir): - os.mkdir(tmpdir) +def recompile(ffi, module_name, preamble, tmpdir='.', **kwds): c_file = os.path.join(tmpdir, module_name + '.c') ext = _get_extension(module_name, c_file, kwds) make_c_source(ffi, module_name, preamble, c_file) From noreply at buildbot.pypy.org Fri Apr 24 13:22:42 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 24 Apr 2015 13:22:42 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: re-add this code Message-ID: <20150424112242.05DD51C069E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1796:eb4454178e77 Date: 2015-04-24 13:22 +0200 http://bitbucket.org/cffi/cffi/changeset/eb4454178e77/ Log: re-add this code diff --git a/_cffi1/recompiler.py b/_cffi1/recompiler.py --- a/_cffi1/recompiler.py +++ b/_cffi1/recompiler.py @@ -527,6 +527,28 @@ _generate_cpy_union_ctx = _generate_cpy_struct_ctx # ---------- + # 'anonymous' declarations. These are produced for anonymous structs + # or unions; the 'name' is obtained by a typedef. + + def _generate_cpy_anonymous_collecttype(self, tp, name): + if isinstance(tp, model.EnumType): + self._generate_cpy_enum_collecttype(tp, name) + else: + self._struct_collecttype(tp) + + def _generate_cpy_anonymous_decl(self, tp, name): + if isinstance(tp, model.EnumType): + self._generate_cpy_enum_decl(tp, name, '') + else: + self._struct_decl(tp, name, 'typedef_' + name) + + def _generate_cpy_anonymous_ctx(self, tp, name): + if isinstance(tp, model.EnumType): + self._generate_cpy_enum_ctx(tp, name, '') + else: + self._struct_ctx(tp, name, 'typedef_' + name) + + # ---------- # constants, declared with "static const ..." def _generate_cpy_const(self, is_int, name, tp=None, category='const', From noreply at buildbot.pypy.org Fri Apr 24 13:22:40 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 24 Apr 2015 13:22:40 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: rename tests Message-ID: <20150424112240.DF7091C069E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1795:f5d8a9cbf03e Date: 2015-04-24 13:18 +0200 http://bitbucket.org/cffi/cffi/changeset/f5d8a9cbf03e/ Log: rename tests diff --git a/_cffi1/test_recompiler.py b/_cffi1/test_recompiler.py --- a/_cffi1/test_recompiler.py +++ b/_cffi1/test_recompiler.py @@ -31,7 +31,7 @@ "(FUNCTION 4)(PRIMITIVE 14)(FUNCTION_END 0)" # sin "(FUNCTION 4)(PRIMITIVE 13)(FUNCTION_END 0)") # cos -def test_use_noop_for_repeated_args(): +def test_type_table_use_noop_for_repeated_args(): check_type_table("double sin(double *, double *);", "(FUNCTION 4)(POINTER 4)(NOOP 1)(FUNCTION_END 0)" "(PRIMITIVE 14)") @@ -39,51 +39,51 @@ "(FUNCTION 3)(POINTER 3)(NOOP 1)(PRIMITIVE 14)" "(FUNCTION_END 0)") -def test_dont_use_noop_for_primitives(): +def test_type_table_dont_use_noop_for_primitives(): check_type_table("double sin(double, double);", "(FUNCTION 1)(PRIMITIVE 14)(PRIMITIVE 14)(FUNCTION_END 0)") -def test_funcptr_as_argument(): +def test_type_table_funcptr_as_argument(): check_type_table("int sin(double(float));", "(FUNCTION 6)(PRIMITIVE 13)(FUNCTION_END 0)" "(FUNCTION 7)(POINTER 0)(FUNCTION_END 0)" "(PRIMITIVE 14)(PRIMITIVE 7)") -def test_variadic_function(): +def test_type_table_variadic_function(): check_type_table("int sin(int, ...);", "(FUNCTION 1)(PRIMITIVE 7)(FUNCTION_END 1)") -def test_array(): +def test_type_table_array(): check_type_table("int a[100];", "(PRIMITIVE 7)(ARRAY 0)(None 100)") -def test_typedef(): +def test_type_table_typedef(): check_type_table("typedef int foo_t;", "(PRIMITIVE 7)") -def test_prebuilt_type(): +def test_type_table_prebuilt_type(): check_type_table("int32_t f(void);", "(FUNCTION 2)(FUNCTION_END 0)(PRIMITIVE 21)") -def test_struct_opaque(): +def test_type_table_struct_opaque(): check_type_table("struct foo_s;", "(STRUCT_UNION 0)") -def test_struct(): +def test_type_table_struct(): check_type_table("struct foo_s { int a; long b; };", "(PRIMITIVE 7)(PRIMITIVE 9)(STRUCT_UNION 0)") -def test_union(): +def test_type_table_union(): check_type_table("union foo_u { int a; long b; };", "(PRIMITIVE 7)(PRIMITIVE 9)(STRUCT_UNION 0)") -def test_struct_used(): +def test_type_table_struct_used(): check_type_table("struct foo_s { int a; long b; }; int f(struct foo_s*);", "(FUNCTION 3)(POINTER 5)(FUNCTION_END 0)" "(PRIMITIVE 7)(PRIMITIVE 9)" "(STRUCT_UNION 0)") -def test_anonymous_struct_with_typedef(): +def test_type_table_anonymous_struct_with_typedef(): check_type_table("typedef struct { int a; long b; } foo_t;", "(STRUCT_UNION 0)(PRIMITIVE 7)(PRIMITIVE 9)") From noreply at buildbot.pypy.org Fri Apr 24 13:40:44 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 24 Apr 2015 13:40:44 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: more sanitization Message-ID: <20150424114044.BA7B21C022E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1797:09a38a35f6b7 Date: 2015-04-24 13:34 +0200 http://bitbucket.org/cffi/cffi/changeset/09a38a35f6b7/ Log: more sanitization diff --git a/_cffi1/recompiler.py b/_cffi1/recompiler.py --- a/_cffi1/recompiler.py +++ b/_cffi1/recompiler.py @@ -702,10 +702,13 @@ return outputfilename def verify(ffi, module_name, preamble, *args, **kwds): + from _cffi1.udir import udir import imp assert module_name not in sys.modules, "module name conflict: %r" % ( module_name,) - outputfilename = recompile(ffi, module_name, preamble, *args, **kwds) + outputfilename = recompile(ffi, module_name, preamble, + tmpdir=str(udir), + *args, **kwds) module = imp.load_dynamic(module_name, outputfilename) # # hack hack hack: copy all *bound methods* from module.ffi back to the diff --git a/_cffi1/test_verify1.py b/_cffi1/test_verify1.py --- a/_cffi1/test_verify1.py +++ b/_cffi1/test_verify1.py @@ -21,12 +21,18 @@ extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion'] class FFI(FFI): + _extra_compile_args = extra_compile_args _verify_counter = 0 def verify(self, preamble='', *args, **kwds): + from _cffi1.udir import udir FFI._verify_counter += 1 return recompiler.verify(self, 'verify%d' % FFI._verify_counter, preamble, *args, - extra_compile_args=extra_compile_args, **kwds) + extra_compile_args=self._extra_compile_args, + tmp=str(udir), **kwds) + +class FFI_warnings_not_error(FFI): + _extra_compile_args = [] class U(object): def __add__(self, other): @@ -53,8 +59,7 @@ def test_missing_function_import_error(): # uses the original FFI that just gives a warning during compilation - import cffi - test_missing_function(ffi=cffi.FFI()) + test_missing_function(ffi=FFI_warnings_not_error()) def test_simple_case(): ffi = FFI() @@ -1855,8 +1860,7 @@ assert repr(ffi.typeof(lib.a)) == "" def test_bug_const_char_ptr_array_2(): - from cffi import FFI # ignore warnings - ffi = FFI() + ffi = FFI_warnings_not_error() # ignore warnings ffi.cdef("""const int a[];""") lib = ffi.verify("""const int a[5];""") assert repr(ffi.typeof(lib.a)) == "" diff --git a/testing/udir.py b/_cffi1/udir.py copy from testing/udir.py copy to _cffi1/udir.py --- a/testing/udir.py +++ b/_cffi1/udir.py @@ -1,3 +1,3 @@ import py -udir = py.path.local.make_numbered_dir(prefix = 'ffi-') +udir = py.path.local.make_numbered_dir(prefix = 'cffi1-') From noreply at buildbot.pypy.org Fri Apr 24 13:40:45 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 24 Apr 2015 13:40:45 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Manual merge of the changes to testing/test_verify.py Message-ID: <20150424114045.CD7571C022E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1798:47b21ee09737 Date: 2015-04-24 13:36 +0200 http://bitbucket.org/cffi/cffi/changeset/47b21ee09737/ Log: Manual merge of the changes to testing/test_verify.py diff --git a/_cffi1/test_verify1.py b/_cffi1/test_verify1.py --- a/_cffi1/test_verify1.py +++ b/_cffi1/test_verify1.py @@ -639,9 +639,9 @@ # case the 'static' is completely ignored. ffi.cdef("static const int AA, BB, CC, DD;") lib = ffi.verify("#define AA 42\n" - "#define BB (-43)\n" - "#define CC (22*2)\n" - "#define DD ((unsigned int)142)\n") + "#define BB (-43) // blah\n" + "#define CC (22*2) /* foobar */\n" + "#define DD ((unsigned int)142) /* foo\nbar */\n") assert lib.AA == 42 assert lib.BB == -43 assert lib.CC == 44 @@ -1208,11 +1208,11 @@ xxx def test_opaque_integer_as_function_result(): - import platform - if platform.machine().startswith('sparc'): - py.test.skip('Breaks horribly on sparc (SIGILL + corrupted stack)') - elif platform.machine() == 'mips64' and sys.maxsize > 2**32: - py.test.skip('Segfaults on mips64el') + #import platform + #if platform.machine().startswith('sparc'): + # py.test.skip('Breaks horribly on sparc (SIGILL + corrupted stack)') + #elif platform.machine() == 'mips64' and sys.maxsize > 2**32: + # py.test.skip('Segfaults on mips64el') # XXX bad abuse of "struct { ...; }". It only works a bit by chance # anyway. XXX think about something better :-( ffi = FFI() @@ -1227,6 +1227,40 @@ h = lib.foo() assert ffi.sizeof(h) == ffi.sizeof("short") +def test_return_partial_struct(): + ffi = FFI() + ffi.cdef(""" + typedef struct { int x; ...; } foo_t; + foo_t foo(void); + """) + lib = ffi.verify(""" + typedef struct { int y, x; } foo_t; + foo_t foo(void) { foo_t r = { 45, 81 }; return r; } + """) + h = lib.foo() + assert ffi.sizeof(h) == 2 * ffi.sizeof("int") + assert h.x == 81 + +def test_take_and_return_partial_structs(): + ffi = FFI() + ffi.cdef(""" + typedef struct { int x; ...; } foo_t; + foo_t foo(foo_t, foo_t); + """) + lib = ffi.verify(""" + typedef struct { int y, x; } foo_t; + foo_t foo(foo_t a, foo_t b) { + foo_t r = { 100, a.x * 5 + b.x * 7 }; + return r; + } + """) + args = ffi.new("foo_t[3]") + args[0].x = 1000 + args[2].x = -498 + h = lib.foo(args[0], args[2]) + assert ffi.sizeof(h) == 2 * ffi.sizeof("int") + assert h.x == 1000 * 5 - 498 * 7 + def test_cannot_name_struct_type(): ffi = FFI() ffi.cdef("typedef struct { int x; } *sp; void foo(sp);") @@ -1628,9 +1662,8 @@ e = py.test.raises(TypeError, ffi.verify, "typedef struct { int x; } foo_t; " "foo_t myfunc(void) { foo_t x = { 42 }; return x; }") - assert str(e.value) in [ - "function myfunc: 'foo_t' is used as result type, but is opaque", - "function myfunc: result type 'foo_t' is opaque"] + assert str(e.value) == ( + "function myfunc: 'foo_t' is used as result type, but is opaque") def test_include(): ffi1 = FFI() From noreply at buildbot.pypy.org Fri Apr 24 13:40:46 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 24 Apr 2015 13:40:46 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: fix tests Message-ID: <20150424114046.D766B1C022E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1799:8b8f76173bdc Date: 2015-04-24 13:41 +0200 http://bitbucket.org/cffi/cffi/changeset/8b8f76173bdc/ Log: fix tests diff --git a/_cffi1/test_verify1.py b/_cffi1/test_verify1.py --- a/_cffi1/test_verify1.py +++ b/_cffi1/test_verify1.py @@ -1,6 +1,7 @@ import sys, math, py -from cffi import FFI, VerificationError, model +from cffi import FFI, VerificationError, VerificationMissing, model from _cffi1 import recompiler +import _cffi_backend lib_m = ['m'] if sys.platform == 'win32': @@ -21,8 +22,10 @@ extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion'] class FFI(FFI): + error = _cffi_backend.FFI.error _extra_compile_args = extra_compile_args _verify_counter = 0 + def verify(self, preamble='', *args, **kwds): from _cffi1.udir import udir FFI._verify_counter += 1 @@ -459,9 +462,9 @@ ...; }; """) - py.test.raises(ffi.error, ffi.sizeof, 'struct foo_s') - py.test.raises(TypeError, ffi.offsetof, 'struct foo_s', 'x') - py.test.raises(TypeError, ffi.new, 'struct foo_s *') + py.test.raises(VerificationMissing, ffi.sizeof, 'struct foo_s') + py.test.raises(VerificationMissing, ffi.offsetof, 'struct foo_s', 'x') + py.test.raises(VerificationMissing, ffi.new, 'struct foo_s *') ffi.verify(""" struct foo_s { int a, b, x, c, d, e; From noreply at buildbot.pypy.org Fri Apr 24 14:05:26 2015 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 24 Apr 2015 14:05:26 +0200 (CEST) Subject: [pypy-commit] pypy vmprof2: try to properly disable codemaps Message-ID: <20150424120526.88F0A1C061E@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof2 Changeset: r76918:a4a719af0754 Date: 2015-04-24 14:05 +0200 http://bitbucket.org/pypy/pypy/changeset/a4a719af0754/ Log: try to properly disable codemaps diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py --- a/rpython/jit/backend/arm/assembler.py +++ b/rpython/jit/backend/arm/assembler.py @@ -586,8 +586,8 @@ assert len(set(inputargs)) == len(inputargs) self.setup(looptoken) - self.codemap_builder.enter_portal_frame(jd_id, unique_id, - self.mc.get_relative_pos()) + #self.codemap_builder.enter_portal_frame(jd_id, unique_id, + # self.mc.get_relative_pos()) frame_info = self.datablockwrapper.malloc_aligned( @@ -662,7 +662,7 @@ assert len(set(inputargs)) == len(inputargs) self.setup(original_loop_token) - self.codemap.inherit_code_from_position(faildescr.adr_jump_offset) + #self.codemap.inherit_code_from_position(faildescr.adr_jump_offset) descr_number = compute_unique_id(faildescr) if log: operations = self._inject_debugging_code(faildescr, operations, @@ -886,8 +886,8 @@ size = self.mc.get_relative_pos() res = self.mc.materialize(self.cpu, allblocks, self.cpu.gc_ll_descr.gcrootmap) - self.cpu.codemap.register_codemap( - self.codemap.get_final_bytecode(res, size)) + #self.cpu.codemap.register_codemap( + # self.codemap.get_final_bytecode(res, size)) return res def update_frame_depth(self, frame_depth): diff --git a/rpython/jit/backend/arm/runner.py b/rpython/jit/backend/arm/runner.py --- a/rpython/jit/backend/arm/runner.py +++ b/rpython/jit/backend/arm/runner.py @@ -50,7 +50,7 @@ def setup_once(self): self.cpuinfo.arch_version = detect_arch_version() self.cpuinfo.hf_abi = detect_hardfloat() - self.codemap.setup() + #self.codemap.setup() self.assembler.setup_once() def finish_once(self): diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py --- a/rpython/jit/backend/llsupport/assembler.py +++ b/rpython/jit/backend/llsupport/assembler.py @@ -130,7 +130,8 @@ self.gcmap_for_finish[0] = r_uint(1) def setup(self, looptoken): - self.codemap_builder = CodemapBuilder() + if self.cpu.HAS_CODEMAP: + self.codemap_builder = CodemapBuilder() self._finish_gcmap = lltype.nullptr(jitframe.GCMAP) def set_debug(self, v): @@ -200,13 +201,15 @@ return fail_descr, target def enter_portal_frame(self, op): - self.codemap_builder.enter_portal_frame(op.getarg(0).getint(), - op.getarg(1).getint(), - self.mc.get_relative_pos()) + if self.cpu.HAS_CODEMAP: + self.codemap_builder.enter_portal_frame(op.getarg(0).getint(), + op.getarg(1).getint(), + self.mc.get_relative_pos()) def leave_portal_frame(self, op): - self.codemap_builder.leave_portal_frame(op.getarg(0).getint(), - self.mc.get_relative_pos()) + if self.cpu.HAS_CODEMAP: + self.codemap_builder.leave_portal_frame(op.getarg(0).getint(), + self.mc.get_relative_pos()) def call_assembler(self, op, guard_op, argloc, vloc, result_loc, tmploc): self._store_force_index(guard_op) diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -23,6 +23,8 @@ class AbstractLLCPU(AbstractCPU): from rpython.jit.metainterp.typesystem import llhelper as ts + HAS_CODEMAP = False + def __init__(self, rtyper, stats, opts, translate_support_code=False, gcdescr=None): assert type(opts) is not bool @@ -49,7 +51,8 @@ else: self._setup_exception_handling_untranslated() self.asmmemmgr = asmmemmgr.AsmMemoryManager() - self.codemap = codemap.CodemapStorage() + if self.HAS_CODEMAP: + self.codemap = codemap.CodemapStorage() self._setup_frame_realloc(translate_support_code) ad = self.gc_ll_descr.getframedescrs(self).arraydescr self.signedarraydescr = ad @@ -80,7 +83,8 @@ pass def finish_once(self): - self.codemap.finish_once() + if self.HAS_CODEMAP: + self.codemap.finish_once() def compile_loop(self, inputargs, operations, looptoken, jd_id=0, unique_id=0, log=True, name='', logger=None): @@ -222,7 +226,8 @@ for rawstart, rawstop in blocks: self.gc_ll_descr.freeing_block(rawstart, rawstop) self.asmmemmgr.free(rawstart, rawstop) - self.codemap.free_asm_block(rawstart, rawstop) + if self.HAS_CODEMAP: + self.codemap.free_asm_block(rawstart, rawstop) def force(self, addr_of_force_token): frame = rffi.cast(jitframe.JITFRAMEPTR, addr_of_force_token) diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -457,8 +457,9 @@ assert len(set(inputargs)) == len(inputargs) self.setup(looptoken) - self.codemap_builder.enter_portal_frame(jd_id, unique_id, - self.mc.get_relative_pos()) + if self.cpu.HAS_CODEMAP: + self.codemap_builder.enter_portal_frame(jd_id, unique_id, + self.mc.get_relative_pos()) frame_info = self.datablockwrapper.malloc_aligned( jitframe.JITFRAMEINFO_SIZE, alignment=WORD) clt.frame_info = rffi.cast(jitframe.JITFRAMEINFOPTR, frame_info) @@ -526,8 +527,9 @@ assert len(set(inputargs)) == len(inputargs) self.setup(original_loop_token) - self.codemap_builder.inherit_code_from_position( - faildescr.adr_jump_offset) + if self.cpu.HAS_CODEMAP: + self.codemap_builder.inherit_code_from_position( + faildescr.adr_jump_offset) self.mc.force_frame_size(DEFAULT_FRAME_BYTES) descr_number = compute_unique_id(faildescr) if log: @@ -692,8 +694,9 @@ size = self.mc.get_relative_pos() res = self.mc.materialize(self.cpu, allblocks, self.cpu.gc_ll_descr.gcrootmap) - self.cpu.codemap.register_codemap( - self.codemap_builder.get_final_bytecode(res, size)) + if self.cpu.HAS_CODEMAP: + self.cpu.codemap.register_codemap( + self.codemap_builder.get_final_bytecode(res, size)) return res def patch_jump_for_descr(self, faildescr, adr_new_target): diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -67,7 +67,8 @@ @rgc.no_release_gil def setup_once(self): self.profile_agent.startup() - self.codemap.setup() + if self.HAS_CODEMAP: + self.codemap.setup() self.assembler.setup_once() @rgc.no_release_gil @@ -158,5 +159,6 @@ CALLEE_SAVE_REGISTERS = [regloc.ebx, regloc.r12, regloc.r13, regloc.r14, regloc.r15] IS_64_BIT = True + HAS_CODEMAP = True CPU = CPU386 diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py --- a/rpython/jit/backend/x86/rx86.py +++ b/rpython/jit/backend/x86/rx86.py @@ -4,6 +4,7 @@ from rpython.rlib.unroll import unrolling_iterable from rpython.rlib.rarithmetic import intmask from rpython.rtyper.lltypesystem import rffi +from rpython.jit.backend.x86.arch import IS_X86_64 BYTE_REG_FLAG = 0x20 NO_BASE_REGISTER = -1 @@ -489,9 +490,10 @@ assert self._frame_size >= self.WORD def check_stack_size_at_ret(self): - assert self._frame_size == self.WORD - if not we_are_translated(): - self._frame_size = None + if IS_X86_64: + assert self._frame_size == self.WORD + if not we_are_translated(): + self._frame_size = None # ------------------------------ MOV ------------------------------ diff --git a/rpython/jit/backend/x86/test/test_assembler.py b/rpython/jit/backend/x86/test/test_assembler.py --- a/rpython/jit/backend/x86/test/test_assembler.py +++ b/rpython/jit/backend/x86/test/test_assembler.py @@ -51,7 +51,8 @@ asmmemmgr_blocks = None cpu = ACTUAL_CPU(None, None) cpu.setup() - cpu.codemap.setup() + if cpu.HAS_CODEMAP: + cpu.codemap.setup() looptoken = FakeToken() asm = cpu.assembler asm.setup_once() From noreply at buildbot.pypy.org Fri Apr 24 14:33:05 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 24 Apr 2015 14:33:05 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: grr Message-ID: <20150424123305.AF3E91C0726@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1800:6f19cda57c3e Date: 2015-04-24 14:02 +0200 http://bitbucket.org/cffi/cffi/changeset/6f19cda57c3e/ Log: grr diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -2251,10 +2251,11 @@ else return convert_to_object_bitfield(data, cf); } + break; case -1: return NULL; default: - ; + break; } } return PyObject_GenericGetAttr((PyObject *)cd, attr); @@ -2284,10 +2285,11 @@ return -1; } } + break; case -1: return -1; default: - ; + break; } } return PyObject_GenericSetAttr((PyObject *)cd, attr, value); From noreply at buildbot.pypy.org Fri Apr 24 14:33:06 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 24 Apr 2015 14:33:06 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Parsing enums Message-ID: <20150424123306.BE8621C0726@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1801:e5783fd730ed Date: 2015-04-24 14:21 +0200 http://bitbucket.org/cffi/cffi/changeset/e5783fd730ed/ Log: Parsing enums diff --git a/_cffi1/parse_c_type.c b/_cffi1/parse_c_type.c --- a/_cffi1/parse_c_type.c +++ b/_cffi1/parse_c_type.c @@ -26,6 +26,7 @@ //TOK__COMPLEX, TOK_CONST, TOK_DOUBLE, + TOK_ENUM, TOK_FLOAT, //TOK__IMAGINARY, TOK_INT, @@ -150,6 +151,9 @@ case 'd': if (tok->size == 6 && !memcmp(p, "double", 6)) tok->kind = TOK_DOUBLE; break; + case 'e': + if (tok->size == 4 && !memcmp(p, "enum", 4)) tok->kind = TOK_ENUM; + break; case 'f': if (tok->size == 5 && !memcmp(p, "float", 5)) tok->kind = TOK_FLOAT; break; @@ -375,6 +379,7 @@ MAKE_SEARCH_FUNC(globals) MAKE_SEARCH_FUNC(struct_unions) MAKE_SEARCH_FUNC(typenames) +MAKE_SEARCH_FUNC(enums) #undef MAKE_SEARCH_FUNC @@ -509,6 +514,7 @@ case TOK_FLOAT: case TOK_STRUCT: case TOK_UNION: + case TOK_ENUM: return parse_error(tok, "invalid combination of types"); case TOK_DOUBLE: @@ -598,6 +604,19 @@ t1 = _CFFI_OP(_CFFI_OP_STRUCT_UNION, n); break; } + case TOK_ENUM: + { + next_token(tok); + if (tok->kind != TOK_IDENTIFIER) + return parse_error(tok, "enum name expected"); + + int n = search_in_enums(tok->info->ctx, tok->p, tok->size); + if (n < 0) + return parse_error(tok, "undefined enum name"); + + t1 = _CFFI_OP(_CFFI_OP_ENUM, n); + break; + } default: return parse_error(tok, "identifier expected"); } diff --git a/_cffi1/test_parse_c_type.py b/_cffi1/test_parse_c_type.py --- a/_cffi1/test_parse_c_type.py +++ b/_cffi1/test_parse_c_type.py @@ -21,6 +21,9 @@ struct_names = ["bar_s", "foo", "foo_", "foo_s", "foo_s1", "foo_s12"] assert struct_names == sorted(struct_names) +enum_names = ["ebar_s", "efoo", "efoo_", "efoo_s", "efoo_s1", "efoo_s12"] +assert enum_names == sorted(enum_names) + identifier_names = ["id", "id0", "id05", "id05b", "tail"] assert identifier_names == sorted(identifier_names) @@ -33,6 +36,13 @@ ctx.struct_unions = ctx_structs ctx.num_struct_unions = len(struct_names) +c_enum_names = [ffi.new("char[]", _n) for _n in enum_names] +ctx_enums = ffi.new("struct _cffi_enum_s[]", len(enum_names)) +for _i in range(len(enum_names)): + ctx_enums[_i].name = c_enum_names[_i] +ctx.enums = ctx_enums +ctx.num_enums = len(enum_names) + c_identifier_names = [ffi.new("char[]", _n) for _n in identifier_names] ctx_identifiers = ffi.new("struct _cffi_typename_s[]", len(identifier_names)) for _i in range(len(identifier_names)): @@ -93,6 +103,7 @@ Func = make_getter('FUNCTION') FuncEnd = make_getter('FUNCTION_END') Struct = make_getter('STRUCT_UNION') +Enum = make_getter('ENUM') Typename = make_getter('TYPENAME') @@ -197,6 +208,12 @@ Prim(lib._CFFI_PRIM_CHAR), OpenArray(4)] +def test_enum(): + for i in range(len(enum_names)): + assert parse("enum %s" % (enum_names[i],)) == ['->', Enum(i)] + assert parse("enum %s*" % (enum_names[i],)) == [Enum(i), + '->', Pointer(0)] + def test_error(): parse_error("short short int", "'short' after another 'short' or 'long'", 6) parse_error("long long long", "'long long long' is too long", 10) From noreply at buildbot.pypy.org Fri Apr 24 14:55:48 2015 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 24 Apr 2015 14:55:48 +0200 (CEST) Subject: [pypy-commit] pypy vmprof2: one missing place Message-ID: <20150424125548.70A8B1C03F7@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof2 Changeset: r76919:afd3d870360e Date: 2015-04-24 14:55 +0200 http://bitbucket.org/pypy/pypy/changeset/afd3d870360e/ Log: one missing place diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py --- a/pypy/module/_vmprof/test/test__vmprof.py +++ b/pypy/module/_vmprof/test/test__vmprof.py @@ -27,6 +27,10 @@ i += 1 # length i += len('pypy') while i < len(s): + if s[i] == '\x03': + break + if s[i] == '\x01': + xxx assert s[i] == '\x02' i += 1 _, size = struct.unpack("ll", s[i:i + 2 * WORD]) diff --git a/rpython/jit/backend/llsupport/asmmemmgr.py b/rpython/jit/backend/llsupport/asmmemmgr.py --- a/rpython/jit/backend/llsupport/asmmemmgr.py +++ b/rpython/jit/backend/llsupport/asmmemmgr.py @@ -319,9 +319,10 @@ assert gcrootmap is not None for pos, mark in self.gcroot_markers: gcrootmap.register_asm_addr(rawstart + pos, mark) - cpu.codemap.register_frame_depth_map(rawstart, rawstart + size, - self.frame_positions, - self.frame_assignments) + if cpu.HAS_CODEMAP: + cpu.codemap.register_frame_depth_map(rawstart, rawstart + size, + self.frame_positions, + self.frame_assignments) self.frame_positions = None self.frame_assignments = None return rawstart From noreply at buildbot.pypy.org Fri Apr 24 15:04:04 2015 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 24 Apr 2015 15:04:04 +0200 (CEST) Subject: [pypy-commit] pypy vmprof2: move those ops to no-side-effect (why they had side effects in the first place?) Message-ID: <20150424130404.50E501C0726@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof2 Changeset: r76920:329a83be5923 Date: 2015-04-24 15:04 +0200 http://bitbucket.org/pypy/pypy/changeset/329a83be5923/ Log: move those ops to no-side-effect (why they had side effects in the first place?) diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -503,6 +503,10 @@ 'MARK_OPAQUE_PTR/1b', # this one has no *visible* side effect, since the virtualizable # must be forced, however we need to execute it anyway + 'DEBUG_MERGE_POINT/*', # debugging only + 'ENTER_PORTAL_FRAME/2', # debugging only + 'LEAVE_PORTAL_FRAME/1', # debugging only + 'JIT_DEBUG/*', # debugging only '_NOSIDEEFFECT_LAST', # ----- end of no_side_effect operations ----- 'INCREMENT_DEBUG_COUNTER/1', @@ -521,10 +525,6 @@ 'UNICODESETITEM/3', 'COND_CALL_GC_WB/1d', # [objptr] (for the write barrier) 'COND_CALL_GC_WB_ARRAY/2d', # [objptr, arrayindex] (write barr. for array) - 'DEBUG_MERGE_POINT/*', # debugging only - 'ENTER_PORTAL_FRAME/2', # debugging only - 'LEAVE_PORTAL_FRAME/1', # debugging only - 'JIT_DEBUG/*', # debugging only 'VIRTUAL_REF_FINISH/2', # removed before it's passed to the backend 'COPYSTRCONTENT/5', # src, dst, srcstart, dststart, length 'COPYUNICODECONTENT/5', From noreply at buildbot.pypy.org Fri Apr 24 15:27:08 2015 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 24 Apr 2015 15:27:08 +0200 (CEST) Subject: [pypy-commit] pypy vmprof2: backend removes operations that have no side effects. Ignore those ops Message-ID: <20150424132708.48C631C069E@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof2 Changeset: r76921:b16af35250ab Date: 2015-04-24 15:27 +0200 http://bitbucket.org/pypy/pypy/changeset/b16af35250ab/ Log: backend removes operations that have no side effects. Ignore those ops explicitely in heap.py diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -293,6 +293,8 @@ opnum == rop.UNICODESETITEM or # no effect on GC struct/array opnum == rop.DEBUG_MERGE_POINT or # no effect whatsoever opnum == rop.JIT_DEBUG or # no effect whatsoever + opnum == rop.ENTER_PORTAL_FRAME or # no effect whatsoever + opnum == rop.LEAVE_PORTAL_FRAME or # no effect whatsoever opnum == rop.COPYSTRCONTENT or # no effect on GC struct/array opnum == rop.COPYUNICODECONTENT): # no effect on GC struct/array return diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -503,10 +503,6 @@ 'MARK_OPAQUE_PTR/1b', # this one has no *visible* side effect, since the virtualizable # must be forced, however we need to execute it anyway - 'DEBUG_MERGE_POINT/*', # debugging only - 'ENTER_PORTAL_FRAME/2', # debugging only - 'LEAVE_PORTAL_FRAME/1', # debugging only - 'JIT_DEBUG/*', # debugging only '_NOSIDEEFFECT_LAST', # ----- end of no_side_effect operations ----- 'INCREMENT_DEBUG_COUNTER/1', @@ -525,6 +521,10 @@ 'UNICODESETITEM/3', 'COND_CALL_GC_WB/1d', # [objptr] (for the write barrier) 'COND_CALL_GC_WB_ARRAY/2d', # [objptr, arrayindex] (write barr. for array) + 'DEBUG_MERGE_POINT/*', # debugging only + 'ENTER_PORTAL_FRAME/2', # debugging only + 'LEAVE_PORTAL_FRAME/1', # debugging only + 'JIT_DEBUG/*', # debugging only 'VIRTUAL_REF_FINISH/2', # removed before it's passed to the backend 'COPYSTRCONTENT/5', # src, dst, srcstart, dststart, length 'COPYUNICODECONTENT/5', From noreply at buildbot.pypy.org Fri Apr 24 15:49:50 2015 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 24 Apr 2015 15:49:50 +0200 (CEST) Subject: [pypy-commit] pypy vmprof2: fix the test Message-ID: <20150424134950.094451C01D6@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof2 Changeset: r76922:1d47005edfcc Date: 2015-04-24 15:38 +0200 http://bitbucket.org/pypy/pypy/changeset/1d47005edfcc/ Log: fix the test diff --git a/rpython/jit/backend/x86/test/test_regloc.py b/rpython/jit/backend/x86/test/test_regloc.py --- a/rpython/jit/backend/x86/test/test_regloc.py +++ b/rpython/jit/backend/x86/test/test_regloc.py @@ -105,13 +105,17 @@ assert ''.join([buf[i] for i in range(length)]) == expected lltype.free(buf, flavor='raw') +class Fake32CodeBlockWrapper(codebuf.MachineCodeBlockWrapper): + def check_stack_size_at_ret(self): + pass + def test_follow_jump_instructions_32(): buf = lltype.malloc(rffi.CCHARP.TO, 80, flavor='raw') raw = rffi.cast(lltype.Signed, buf) - mc = codebuf.MachineCodeBlockWrapper(); mc.WORD = 4; mc.relocations = [] + mc = Fake32CodeBlockWrapper(); mc.WORD = 4; mc.relocations = [] mc.RET() mc.copy_to_raw_memory(raw) - mc = codebuf.MachineCodeBlockWrapper(); mc.WORD = 4; mc.relocations = [] + mc = Fake32CodeBlockWrapper(); mc.WORD = 4; mc.relocations = [] assert follow_jump(raw) == raw mc.JMP(imm(raw)) mc.copy_to_raw_memory(raw + 20) @@ -120,7 +124,7 @@ assert buf[22] == '\xFF' assert buf[23] == '\xFF' assert buf[24] == '\xFF' - mc = codebuf.MachineCodeBlockWrapper(); mc.WORD = 4; mc.relocations = [] + mc = Fake32CodeBlockWrapper(); mc.WORD = 4; mc.relocations = [] assert follow_jump(raw + 20) == raw mc.JMP(imm(raw)) mc.copy_to_raw_memory(raw + 40) From noreply at buildbot.pypy.org Fri Apr 24 15:49:51 2015 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 24 Apr 2015 15:49:51 +0200 (CEST) Subject: [pypy-commit] pypy vmprof2: fix test_asmmemmgr Message-ID: <20150424134951.33CCF1C01D6@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof2 Changeset: r76923:b37b7f15984b Date: 2015-04-24 15:39 +0200 http://bitbucket.org/pypy/pypy/changeset/b37b7f15984b/ Log: fix test_asmmemmgr diff --git a/rpython/jit/backend/llsupport/test/test_asmmemmgr.py b/rpython/jit/backend/llsupport/test/test_asmmemmgr.py --- a/rpython/jit/backend/llsupport/test/test_asmmemmgr.py +++ b/rpython/jit/backend/llsupport/test/test_asmmemmgr.py @@ -173,6 +173,7 @@ # gcrootmap = FakeGcRootMap() allblocks = [] + self.HAS_CODEMAP = False rawstart = mc.materialize(self, allblocks, gcrootmap) p = rffi.cast(rffi.CArrayPtr(lltype.Char), rawstart) assert p[0] == 'X' From noreply at buildbot.pypy.org Fri Apr 24 15:49:52 2015 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 24 Apr 2015 15:49:52 +0200 (CEST) Subject: [pypy-commit] pypy vmprof2: try a different approach to building against libunwind Message-ID: <20150424134952.65D321C01D6@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof2 Changeset: r76924:4ece3460b60d Date: 2015-04-24 15:49 +0200 http://bitbucket.org/pypy/pypy/changeset/4ece3460b60d/ Log: try a different approach to building against libunwind diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py --- a/pypy/module/_vmprof/interp_vmprof.py +++ b/pypy/module/_vmprof/interp_vmprof.py @@ -27,7 +27,7 @@ include_dirs = [SRC], includes = ['vmprof.h', 'trampoline.h'], separate_module_files = [SRC.join('trampoline.asmgcc.s')], - libraries = ['unwind'], + link_extra = ['-Wl,-Bstatic', '-lunwind', '-Wl,-Bdynamic', '-llzma'], post_include_bits=[""" void pypy_vmprof_init(void); From noreply at buildbot.pypy.org Fri Apr 24 16:12:29 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 24 Apr 2015 16:12:29 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: for now, we don't check the cdef value given in a "#define X Y" Message-ID: <20150424141229.C3F3A1C061E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1803:ba730ccef0e0 Date: 2015-04-24 15:24 +0200 http://bitbucket.org/cffi/cffi/changeset/ba730ccef0e0/ Log: for now, we don't check the cdef value given in a "#define X Y" diff --git a/_cffi1/recompiler.py b/_cffi1/recompiler.py --- a/_cffi1/recompiler.py +++ b/_cffi1/recompiler.py @@ -563,9 +563,7 @@ # ---------- # constants, declared with "static const ..." - def _generate_cpy_const(self, is_int, name, tp=None, category='const', - check_value=None): - assert check_value is None # XXX + def _generate_cpy_const(self, is_int, name, tp=None, category='const'): prnt = self._prnt funcname = '_cffi_%s_%s' % (category, name) if is_int: @@ -603,7 +601,7 @@ # ---------- # enums - + def _generate_cpy_enum_collecttype(self, tp, name): self._do_collect_type(tp) @@ -614,11 +612,9 @@ pass def _generate_cpy_macro_decl(self, tp, name): - if tp == '...': - check_value = None - else: - check_value = tp # an integer - self._generate_cpy_const(True, name, check_value=check_value) + # for now, we ignore the value (if != ',,,') given in the cdef + # and always trust the value coming from the C compiler + self._generate_cpy_const(True, name) def _generate_cpy_macro_ctx(self, tp, name): self._lsts["global"].append( diff --git a/_cffi1/test_recompiler.py b/_cffi1/test_recompiler.py --- a/_cffi1/test_recompiler.py +++ b/_cffi1/test_recompiler.py @@ -1,4 +1,4 @@ -import py +import sys, py from cffi import FFI from _cffi1 import recompiler @@ -160,6 +160,18 @@ assert lib.FOOBAR == -6912 py.test.raises(AttributeError, "lib.FOOBAR = 2") +def test_macro_check_value_ok(): + ffi = FFI() + ffi.cdef("#define FOOBAR 42") + lib = verify(ffi, 'test_macro_check_value_ok', "#define FOOBAR 42") + assert lib.FOOBAR == 42 + +def test_macro_check_value_fail(): + ffi = FFI() + ffi.cdef("#define FOOBAR 42") + lib = verify(ffi, 'test_macro_check_value_fail', "#define FOOBAR 43") + assert lib.FOOBAR == 43 # for now, we don't check the cdef value + def test_constant(): ffi = FFI() ffi.cdef("static const int FOOBAR;") @@ -251,6 +263,17 @@ assert ffi1.typeof("void(*)(struct foo_s*)") is not ( ffi2.typeof("void(*)(struct foo_s*)")) +def test_verify_enum(): + py.test.skip("in-progress") + ffi = FFI() + ffi.cdef("""enum e1 { B1, A1, ... };""") + lib = verify(ffi, 'test_verify_enum', + "enum e1 { A1, B1, C1=%d };" % sys.maxint) + ffi.typeof("enum e1") + assert lib.A1 == 0 + assert lib.B1 == 0 + assert ffi.sizeof("enum e1") == ffi.sizeof("long") + def test_dotdotdot_length_of_array_field(): ffi = FFI() ffi.cdef("struct foo_s { int a[...]; int b[...]; };") From noreply at buildbot.pypy.org Fri Apr 24 16:12:28 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 24 Apr 2015 16:12:28 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: enums in the type table Message-ID: <20150424141228.B3D641C061E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1802:5acd08cb3958 Date: 2015-04-24 14:39 +0200 http://bitbucket.org/cffi/cffi/changeset/5acd08cb3958/ Log: enums in the type table diff --git a/_cffi1/parse_c_type.h b/_cffi1/parse_c_type.h --- a/_cffi1/parse_c_type.h +++ b/_cffi1/parse_c_type.h @@ -90,7 +90,7 @@ struct _cffi_enum_s { const char *name; - _cffi_opcode_t integer_type_op; + int size_and_sign; }; struct _cffi_typename_s { diff --git a/_cffi1/recompiler.py b/_cffi1/recompiler.py --- a/_cffi1/recompiler.py +++ b/_cffi1/recompiler.py @@ -42,14 +42,20 @@ self.cffi_types.append('LEN') # placeholder assert None not in self._typesdict.values() # - # collect all structs and unions + # collect all structs and unions and enums self._struct_unions = {} + self._enums = {} for tp in all_decls: if isinstance(tp, model.StructOrUnion): self._struct_unions[tp] = None + elif isinstance(tp, model.EnumType): + self._enums[tp] = None for i, tp in enumerate(sorted(self._struct_unions, key=lambda tp: tp.name)): self._struct_unions[tp] = i + for i, tp in enumerate(sorted(self._enums, + key=lambda tp: tp.name)): + self._enums[tp] = i # # emit all bytecode sequences now for tp in all_decls: @@ -89,7 +95,7 @@ step_name)) except AttributeError: raise ffiplatform.VerificationError( - "not implemented in verify(): %r" % name) + "not implemented in recompile(): %r" % name) try: method(tp, realname) except Exception as e: @@ -168,6 +174,12 @@ assert i < len(lst) assert lst[i].startswith(' { "%s"' % tp.name) assert len(lst) == len(self._struct_unions) + # same with enums + lst = self._lsts["enum"] + for tp, i in self._enums.items(): + assert i < len(lst) + assert lst[i].startswith(' { "%s"' % tp.name) + assert len(lst) == len(self._enums) # # the declaration of '_cffi_type_context' prnt('static const struct _cffi_type_context_s _cffi_type_context = {') @@ -590,6 +602,12 @@ ' { "%s", _cffi_const_%s, %s },' % (name, name, type_op)) # ---------- + # enums + + def _generate_cpy_enum_collecttype(self, tp, name): + self._do_collect_type(tp) + + # ---------- # macros: for now only for integers def _generate_cpy_macro_collecttype(self, tp, name): @@ -679,8 +697,12 @@ def _emit_bytecode_StructType(self, tp, index): struct_index = self._struct_unions[tp] self.cffi_types[index] = CffiOp(OP_STRUCT_UNION, struct_index) + _emit_bytecode_UnionType = _emit_bytecode_StructType - _emit_bytecode_UnionType = _emit_bytecode_StructType + def _emit_bytecode_EnumType(self, tp, index): + enum_index = self._enums[tp] + self.cffi_types[index] = CffiOp(OP_ENUM, enum_index) + def make_c_source(ffi, module_name, preamble, target_c_file): recompiler = Recompiler(ffi, module_name) diff --git a/_cffi1/test_recompiler.py b/_cffi1/test_recompiler.py --- a/_cffi1/test_recompiler.py +++ b/_cffi1/test_recompiler.py @@ -87,6 +87,10 @@ check_type_table("typedef struct { int a; long b; } foo_t;", "(STRUCT_UNION 0)(PRIMITIVE 7)(PRIMITIVE 9)") +def test_type_table_enum(): + check_type_table("enum foo_e { AA, BB, ... };", + "(ENUM 0)") + def test_math_sin(): import math From noreply at buildbot.pypy.org Fri Apr 24 16:12:30 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 24 Apr 2015 16:12:30 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Enums, first complete passing tests Message-ID: <20150424141230.D30491C061E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1804:fff6ae4c61b2 Date: 2015-04-24 16:00 +0200 http://bitbucket.org/cffi/cffi/changeset/fff6ae4c61b2/ Log: Enums, first complete passing tests diff --git a/_cffi1/_cffi_include.h b/_cffi1/_cffi_include.h --- a/_cffi1/_cffi_include.h +++ b/_cffi1/_cffi_include.h @@ -161,6 +161,13 @@ #define _cffi_array_len(array) (sizeof(array) / sizeof((array)[0])) +#define _cffi_prim_int(size, sign) \ + ((size) == 1 ? ((sign) ? _CFFI_PRIM_INT8 : _CFFI_PRIM_UINT8) : \ + (size) == 2 ? ((sign) ? _CFFI_PRIM_INT16 : _CFFI_PRIM_UINT16) : \ + (size) == 4 ? ((sign) ? _CFFI_PRIM_INT32 : _CFFI_PRIM_UINT32) : \ + (size) == 8 ? ((sign) ? _CFFI_PRIM_INT64 : _CFFI_PRIM_UINT64) : \ + 0) + static int _cffi_init(void) { diff --git a/_cffi1/parse_c_type.h b/_cffi1/parse_c_type.h --- a/_cffi1/parse_c_type.h +++ b/_cffi1/parse_c_type.h @@ -90,7 +90,7 @@ struct _cffi_enum_s { const char *name; - int size_and_sign; + int type_prim; /* _CFFI_PRIM_xxx */ }; struct _cffi_typename_s { diff --git a/_cffi1/realize_c_type.c b/_cffi1/realize_c_type.c --- a/_cffi1/realize_c_type.c +++ b/_cffi1/realize_c_type.c @@ -360,6 +360,18 @@ break; } + case _CFFI_OP_ENUM: + { + const struct _cffi_enum_s *e; + + e = &builder->ctx.enums[_CFFI_GETARG(op)]; + x = all_primitives[e->type_prim]; + if (x == NULL) + x = build_primitive_type(e->type_prim); + Py_XINCREF(x); + break; + } + case _CFFI_OP_FUNCTION: { PyObject *fargs; diff --git a/_cffi1/recompiler.py b/_cffi1/recompiler.py --- a/_cffi1/recompiler.py +++ b/_cffi1/recompiler.py @@ -550,13 +550,13 @@ def _generate_cpy_anonymous_decl(self, tp, name): if isinstance(tp, model.EnumType): - self._generate_cpy_enum_decl(tp, name, '') + self._generate_cpy_enum_decl(tp) else: self._struct_decl(tp, name, 'typedef_' + name) def _generate_cpy_anonymous_ctx(self, tp, name): if isinstance(tp, model.EnumType): - self._generate_cpy_enum_ctx(tp, name, '') + self._enum_ctx(tp, name) else: self._struct_ctx(tp, name, 'typedef_' + name) @@ -589,15 +589,19 @@ is_int = isinstance(tp, model.PrimitiveType) and tp.is_integer_type() self._generate_cpy_const(is_int, name, tp) + def _generate_const_int_ctx(self, name): + self._lsts["global"].append( + ' { "%s", _cffi_const_%s, _CFFI_OP(_CFFI_OP_CONSTANT_INT, 0) },' % + (name, name)) + def _generate_cpy_constant_ctx(self, tp, name): - is_int = isinstance(tp, model.PrimitiveType) and tp.is_integer_type() - if not is_int: + if isinstance(tp, model.PrimitiveType) and tp.is_integer_type(): + self._generate_const_int_ctx(name) + else: type_index = self._typesdict[tp] type_op = '_CFFI_OP(_CFFI_OP_CONSTANT, %d)' % type_index - else: - type_op = '_CFFI_OP(_CFFI_OP_CONSTANT_INT, 0)' - self._lsts["global"].append( - ' { "%s", _cffi_const_%s, %s },' % (name, name, type_op)) + self._lsts["global"].append( + ' { "%s", _cffi_const_%s, %s },' % (name, name, type_op)) # ---------- # enums @@ -605,6 +609,29 @@ def _generate_cpy_enum_collecttype(self, tp, name): self._do_collect_type(tp) + def _generate_cpy_enum_decl(self, tp, name=None): + for enumerator in tp.enumerators: + self._generate_cpy_const(True, enumerator) + + def _enum_ctx(self, tp, cname): + for enumerator in tp.enumerators: + self._generate_const_int_ctx(enumerator) + if cname is not None: + size = "sizeof(%s)" % cname + signed = "((%s)-1) <= 0" % cname + prim = "_cffi_prim_int(%s, %s)" % (size, signed) + else: + size = xxxx + self._lsts["enum"].append( + ' { "%s", %s },' % (tp.name, prim)) + + def _generate_cpy_enum_ctx(self, tp, name): + if tp.has_c_name(): + cname = tp.get_c_name('') + else: + cname = None + self._enum_ctx(tp, cname) + # ---------- # macros: for now only for integers diff --git a/_cffi1/test_recompiler.py b/_cffi1/test_recompiler.py --- a/_cffi1/test_recompiler.py +++ b/_cffi1/test_recompiler.py @@ -264,16 +264,20 @@ ffi2.typeof("void(*)(struct foo_s*)")) def test_verify_enum(): - py.test.skip("in-progress") ffi = FFI() - ffi.cdef("""enum e1 { B1, A1, ... };""") + ffi.cdef("""enum e1 { B1, A1, ... }; enum e2 { B2, A2, ... };""") lib = verify(ffi, 'test_verify_enum', - "enum e1 { A1, B1, C1=%d };" % sys.maxint) + "enum e1 { A1, B1, C1=%d };" % sys.maxint + + "enum e2 { A2, B2, C2 };") ffi.typeof("enum e1") + ffi.typeof("enum e2") assert lib.A1 == 0 - assert lib.B1 == 0 + assert lib.B1 == 1 + assert lib.A2 == 0 + assert lib.B2 == 1 assert ffi.sizeof("enum e1") == ffi.sizeof("long") - + assert ffi.sizeof("enum e2") == ffi.sizeof("int") + def test_dotdotdot_length_of_array_field(): ffi = FFI() ffi.cdef("struct foo_s { int a[...]; int b[...]; };") @@ -341,3 +345,18 @@ "typedef struct { int a; long b; } *foo_t;") p = ffi.new("foo_t", {'b': 42}) assert p.b == 42 + +def test_verify_anonymous_enum_with_typedef(): + ffi = FFI() + ffi.cdef("typedef enum { AA, ... } e1;") + lib = verify(ffi, 'test_verify_anonymous_enum_with_typedef1', + "typedef enum { BB, CC, AA } e1;") + assert lib.AA == 2 + assert ffi.sizeof("e1") == ffi.sizeof("int") + # + ffi = FFI() + ffi.cdef("typedef enum { AA=%d } e1;" % sys.maxint) + lib = verify(ffi, 'test_verify_anonymous_enum_with_typedef2', + "typedef enum { AA=%d } e1;" % sys.maxint) + assert lib.AA == sys.maxint + assert ffi.sizeof("e1") == ffi.sizeof("long") From noreply at buildbot.pypy.org Fri Apr 24 16:31:45 2015 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 24 Apr 2015 16:31:45 +0200 (CEST) Subject: [pypy-commit] pypy numpy-fixes: test for nan in complex.sign Message-ID: <20150424143145.77CE51C06F5@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: numpy-fixes Changeset: r76927:43ccc6f8802a Date: 2015-04-24 17:20 +0300 http://bitbucket.org/pypy/pypy/changeset/43ccc6f8802a/ Log: test for nan in complex.sign diff --git a/pypy/module/micronumpy/test/test_complex.py b/pypy/module/micronumpy/test/test_complex.py --- a/pypy/module/micronumpy/test/test_complex.py +++ b/pypy/module/micronumpy/test/test_complex.py @@ -331,6 +331,12 @@ complex(float('nan'), 0)], dtype=complex)) == \ [False, True, True, False, False]).all() + def test_sign_for_complex_nan(self): + from numpy import array, nan, sign, isnan + C = array([nan], dtype=complex) + res = sign(C) + assert isnan(res.real) + assert res.imag == 0+0j def test_square(self): from numpy import square diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -1347,15 +1347,17 @@ sign of complex number could be either the point closest to the unit circle or {-1,0,1}, for compatability with numpy we choose the latter ''' + if rfloat.isnan(v[0]) or rfloat.isnan(v[1]): + return rfloat.NAN, 0 if v[0] == 0.0: if v[1] == 0: - return 0,0 + return 0, 0 if v[1] > 0: - return 1,0 - return -1,0 + return 1, 0 + return -1, 0 if v[0] > 0: - return 1,0 - return -1,0 + return 1, 0 + return -1, 0 def fmax(self, v1, v2): if self.ge(v1, v2) or self.isnan(v2): From noreply at buildbot.pypy.org Fri Apr 24 16:31:44 2015 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 24 Apr 2015 16:31:44 +0200 (CEST) Subject: [pypy-commit] pypy numpy-fixes: a branch to fix numpy test suite failures with "Arrays are not equal" message Message-ID: <20150424143144.59EC41C06F5@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: numpy-fixes Changeset: r76926:6981237fb078 Date: 2015-04-24 17:03 +0300 http://bitbucket.org/pypy/pypy/changeset/6981237fb078/ Log: a branch to fix numpy test suite failures with "Arrays are not equal" message diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py --- a/pypy/module/micronumpy/concrete.py +++ b/pypy/module/micronumpy/concrete.py @@ -323,7 +323,7 @@ def __exit__(self, typ, value, traceback): keepalive_until_here(self) - + def get_buffer(self, space, readonly): return ArrayBuffer(self, readonly) From noreply at buildbot.pypy.org Fri Apr 24 16:31:43 2015 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 24 Apr 2015 16:31:43 +0200 (CEST) Subject: [pypy-commit] pypy numpy-fixes: fix d198d926afb8 Message-ID: <20150424143143.398801C06F5@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: numpy-fixes Changeset: r76925:cfb31de4972f Date: 2015-04-24 16:27 +0300 http://bitbucket.org/pypy/pypy/changeset/cfb31de4972f/ Log: fix d198d926afb8 diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py --- a/pypy/module/micronumpy/concrete.py +++ b/pypy/module/micronumpy/concrete.py @@ -11,7 +11,7 @@ from pypy.module.micronumpy.iterators import ArrayIter from pypy.module.micronumpy.strides import (Chunk, Chunks, NewAxisChunk, RecordChunk, calc_strides, calc_new_strides, shape_agreement, - calculate_broadcast_strides, calc_backstrides) + calculate_broadcast_strides, calc_backstrides, calc_start) from rpython.rlib.objectmodel import keepalive_until_here from rpython.rtyper.annlowlevel import cast_gcref_to_instance from pypy.interpreter.baseobjspace import W_Root @@ -328,8 +328,11 @@ return ArrayBuffer(self, readonly) def astype(self, space, dtype): - strides, backstrides = calc_strides(self.get_shape(), dtype, - self.order) + # we want to create a new array, but must respect the strides + # in self. So find a factor of the itemtype.elsize, and use this + factor = float(dtype.elsize) / self.dtype.elsize + strides = [int(factor*s) for s in self.get_strides()] + backstrides = [int(factor*s) for s in self.get_backstrides()] impl = ConcreteArray(self.get_shape(), dtype, self.order, strides, backstrides) loop.setslice(space, impl.get_shape(), impl, self) @@ -426,8 +429,9 @@ gcstruct = _create_objectstore(storage, length, dtype.elsize) else: storage = dtype.itemtype.malloc(length * dtype.elsize, zero=zero) + start = calc_start(shape, strides) ConcreteArrayNotOwning.__init__(self, shape, dtype, order, strides, backstrides, - storage) + storage, start=start) self.gcstruct = gcstruct def __del__(self): diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py --- a/pypy/module/micronumpy/strides.py +++ b/pypy/module/micronumpy/strides.py @@ -429,6 +429,17 @@ n_old_elems_to_use *= old_shape[oldI] return new_strides[:] +def calc_start(shape, strides): + ''' Strides can be negative for non-contiguous data. + Calculate the appropriate positive starting position so + the indexing still works properly + ''' + start = 0 + for i in range(len(shape)): + if strides[i] < 0: + start -= strides[i] * (shape[i] - 1) + return start + @jit.unroll_safe def is_c_contiguous(arr): shape = arr.get_shape() diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -2169,6 +2169,7 @@ def test_astype(self): from numpy import array, arange + import gc b = array(1).astype(float) assert b == 1 assert b.dtype == float @@ -2182,8 +2183,15 @@ assert (b == [False, True, True]).all() assert b.dtype == 'bool' + a = arange(11)[::-1] + b = a.astype('int32') + assert (b == a).all() + del b + gc.collect() + a = arange(6, dtype='f4').reshape(2,3) - b = a.astype('i4') + b = a.T.astype('i4') + assert (a.T.strides == b.strides) a = array('x').astype('S3').dtype assert a.itemsize == 3 diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -28,6 +28,20 @@ log2e = 1. / log2 log10 = math.log(10) +''' +if not we_are_translated(): + _raw_storage_setitem_unaligned = raw_storage_setitem_unaligned + _raw_storage_getitem_unaligned = raw_storage_getitem_unaligned + def raw_storage_setitem_unaligned(storage, offset, value): + assert offset >=0 + assert offset < storage._obj.getlength() + return _raw_storage_setitem_unaligned(storage, offset, value) + + def raw_storage_getitem_unaligned(T, storage, offset): + assert offset >=0 + assert offset < storage._obj.getlength() + return _raw_storage_getitem_unaligned(T, storage, offset) +''' def simple_unary_op(func): specialize.argtype(1)(func) From noreply at buildbot.pypy.org Fri Apr 24 17:15:19 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 24 Apr 2015 17:15:19 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: progress on enums Message-ID: <20150424151519.10DF51C04C7@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1805:7231fd7eae98 Date: 2015-04-24 17:15 +0200 http://bitbucket.org/cffi/cffi/changeset/7231fd7eae98/ Log: progress on enums diff --git a/_cffi1/_cffi_include.h b/_cffi1/_cffi_include.h --- a/_cffi1/_cffi_include.h +++ b/_cffi1/_cffi_include.h @@ -162,10 +162,12 @@ #define _cffi_array_len(array) (sizeof(array) / sizeof((array)[0])) #define _cffi_prim_int(size, sign) \ - ((size) == 1 ? ((sign) ? _CFFI_PRIM_INT8 : _CFFI_PRIM_UINT8) : \ - (size) == 2 ? ((sign) ? _CFFI_PRIM_INT16 : _CFFI_PRIM_UINT16) : \ - (size) == 4 ? ((sign) ? _CFFI_PRIM_INT32 : _CFFI_PRIM_UINT32) : \ - (size) == 8 ? ((sign) ? _CFFI_PRIM_INT64 : _CFFI_PRIM_UINT64) : \ + ((size) == sizeof(int) ? ((sign) ? _CFFI_PRIM_INT : _CFFI_PRIM_UINT) : \ + (size) == sizeof(long)? ((sign) ? _CFFI_PRIM_LONG : _CFFI_PRIM_ULONG) : \ + (size) == 1 ? ((sign) ? _CFFI_PRIM_INT8 : _CFFI_PRIM_UINT8) : \ + (size) == 2 ? ((sign) ? _CFFI_PRIM_INT16 : _CFFI_PRIM_UINT16) : \ + (size) == 4 ? ((sign) ? _CFFI_PRIM_INT32 : _CFFI_PRIM_UINT32) : \ + (size) == 8 ? ((sign) ? _CFFI_PRIM_INT64 : _CFFI_PRIM_UINT64) : \ 0) diff --git a/_cffi1/parse_c_type.h b/_cffi1/parse_c_type.h --- a/_cffi1/parse_c_type.h +++ b/_cffi1/parse_c_type.h @@ -90,7 +90,8 @@ struct _cffi_enum_s { const char *name; - int type_prim; /* _CFFI_PRIM_xxx */ + int type_index; // -> _cffi_types, on a OP_ENUM + int type_prim; // _CFFI_PRIM_xxx }; struct _cffi_typename_s { diff --git a/_cffi1/realize_c_type.c b/_cffi1/realize_c_type.c --- a/_cffi1/realize_c_type.c +++ b/_cffi1/realize_c_type.c @@ -11,6 +11,10 @@ static PyObject *build_primitive_type(int num); /* forward */ +#define get_primitive_type(num) \ + (all_primitives[num] != NULL ? all_primitives[num] \ + : build_primitive_type(num)) + static int init_global_types_dict(PyObject *ffi_type_dict) { int err; @@ -20,7 +24,7 @@ if (global_types_dict == NULL) return -1; - ct = build_primitive_type(_CFFI_PRIM_VOID); // 'void' + ct = get_primitive_type(_CFFI_PRIM_VOID); // 'void' if (ct == NULL) return -1; if (PyDict_SetItemString(global_types_dict, @@ -260,9 +264,7 @@ switch (_CFFI_GETOP(op)) { case _CFFI_OP_PRIMITIVE: - x = all_primitives[_CFFI_GETARG(op)]; - if (x == NULL) - x = build_primitive_type(_CFFI_GETARG(op)); + x = get_primitive_type(_CFFI_GETARG(op)); Py_XINCREF(x); break; @@ -334,7 +336,7 @@ ct->ct_extra = builder; } - /* Update the "primary" OP_STRUCT_OR_UNION slot, which + /* Update the "primary" OP_STRUCT_UNION slot, which may be the same or a different slot than the "current" one */ assert((((uintptr_t)x) & 1) == 0); assert(builder->ctx.types[s->type_index] == op2); @@ -363,12 +365,40 @@ case _CFFI_OP_ENUM: { const struct _cffi_enum_s *e; + _cffi_opcode_t op2; e = &builder->ctx.enums[_CFFI_GETARG(op)]; - x = all_primitives[e->type_prim]; - if (x == NULL) - x = build_primitive_type(e->type_prim); - Py_XINCREF(x); + op2 = builder->ctx.types[e->type_index]; + if ((((uintptr_t)op2) & 1) == 0) { + x = (PyObject *)op2; + Py_INCREF(x); + } + else { + PyObject *basetd = get_primitive_type(e->type_prim); + if (basetd == NULL) + return NULL; + + PyObject *args = Py_BuildValue("(s()()O)", e->name, basetd); + if (args == NULL) + return NULL; + + x = b_new_enum_type(NULL, args); + Py_DECREF(args); + if (x == NULL) + return NULL; + + /* Update the "primary" _CFFI_OP_ENUM slot, which + may be the same or a different slot than the "current" one */ + assert((((uintptr_t)x) & 1) == 0); + assert(builder->ctx.types[e->type_index] == op2); + Py_INCREF(x); + builder->ctx.types[e->type_index] = x; + + /* Done, leave without updating the "current" slot because + it may be done already above. If not, never mind, the + next call to realize_c_type() will do it. */ + return x; + } break; } diff --git a/_cffi1/recompiler.py b/_cffi1/recompiler.py --- a/_cffi1/recompiler.py +++ b/_cffi1/recompiler.py @@ -589,19 +589,14 @@ is_int = isinstance(tp, model.PrimitiveType) and tp.is_integer_type() self._generate_cpy_const(is_int, name, tp) - def _generate_const_int_ctx(self, name): - self._lsts["global"].append( - ' { "%s", _cffi_const_%s, _CFFI_OP(_CFFI_OP_CONSTANT_INT, 0) },' % - (name, name)) - def _generate_cpy_constant_ctx(self, tp, name): if isinstance(tp, model.PrimitiveType) and tp.is_integer_type(): - self._generate_const_int_ctx(name) + type_op = '_CFFI_OP(_CFFI_OP_CONSTANT_INT, 0)' else: type_index = self._typesdict[tp] type_op = '_CFFI_OP(_CFFI_OP_CONSTANT, %d)' % type_index - self._lsts["global"].append( - ' { "%s", _cffi_const_%s, %s },' % (name, name, type_op)) + self._lsts["global"].append( + ' { "%s", _cffi_const_%s, %s },' % (name, name, type_op)) # ---------- # enums @@ -614,8 +609,13 @@ self._generate_cpy_const(True, enumerator) def _enum_ctx(self, tp, cname): + type_index = self._typesdict[tp] + type_op = '_CFFI_OP(_CFFI_OP_ENUM, %d)' % type_index for enumerator in tp.enumerators: - self._generate_const_int_ctx(enumerator) + self._lsts["global"].append( + ' { "%s", _cffi_const_%s, %s },' % (enumerator, enumerator, + type_op)) + # if cname is not None: size = "sizeof(%s)" % cname signed = "((%s)-1) <= 0" % cname @@ -623,7 +623,7 @@ else: size = xxxx self._lsts["enum"].append( - ' { "%s", %s },' % (tp.name, prim)) + ' { "%s", %d, %s },' % (tp.name, type_index, prim)) def _generate_cpy_enum_ctx(self, tp, name): if tp.has_c_name(): From noreply at buildbot.pypy.org Fri Apr 24 17:48:42 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 24 Apr 2015 17:48:42 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Add to the recompiler the value of the elements that "belong" to an Message-ID: <20150424154842.DF7EB1C01D6@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1806:4adfe9dbfcbf Date: 2015-04-24 17:49 +0200 http://bitbucket.org/cffi/cffi/changeset/4adfe9dbfcbf/ Log: Add to the recompiler the value of the elements that "belong" to an enum (even though they don't really belong there in the C sense) diff --git a/_cffi1/lib_obj.c b/_cffi1/lib_obj.c --- a/_cffi1/lib_obj.c +++ b/_cffi1/lib_obj.c @@ -165,23 +165,11 @@ break; case _CFFI_OP_CONSTANT_INT: + case _CFFI_OP_ENUM: { /* a constant integer whose value, in an "unsigned long long", is obtained by calling the function at g->address */ - unsigned long long value; - int neg = ((int(*)(unsigned long long*))g->address)(&value); - if (!neg) { - if (value <= (unsigned long long)LONG_MAX) - x = PyInt_FromLong((long)value); - else - x = PyLong_FromUnsignedLongLong(value); - } - else { - if ((long long)value >= (long long)LONG_MIN) - x = PyInt_FromLong((long)value); - else - x = PyLong_FromLongLong((long long)value); - } + x = realize_global_int(g); break; } diff --git a/_cffi1/parse_c_type.h b/_cffi1/parse_c_type.h --- a/_cffi1/parse_c_type.h +++ b/_cffi1/parse_c_type.h @@ -92,6 +92,7 @@ const char *name; int type_index; // -> _cffi_types, on a OP_ENUM int type_prim; // _CFFI_PRIM_xxx + const char *enumerators; // comma-delimited string }; struct _cffi_typename_s { diff --git a/_cffi1/realize_c_type.c b/_cffi1/realize_c_type.c --- a/_cffi1/realize_c_type.c +++ b/_cffi1/realize_c_type.c @@ -214,6 +214,25 @@ return x; } +static PyObject *realize_global_int(const struct _cffi_global_s *g) +{ + PyObject *x; + unsigned long long value; + int neg = ((int(*)(unsigned long long*))g->address)(&value); + if (!neg) { + if (value <= (unsigned long long)LONG_MAX) + x = PyInt_FromLong((long)value); + else + x = PyLong_FromUnsignedLongLong(value); + } + else { + if ((long long)value >= (long long)LONG_MIN) + x = PyInt_FromLong((long)value); + else + x = PyLong_FromLongLong((long long)value); + } + return x; +} static PyObject * _realize_c_type_or_func(builder_c_t *builder, @@ -378,7 +397,51 @@ if (basetd == NULL) return NULL; - PyObject *args = Py_BuildValue("(s()()O)", e->name, basetd); + PyObject *enumerators = NULL, *enumvalues = NULL, *tmp; + Py_ssize_t i, j, n = 0; + const char *p; + const struct _cffi_global_s *g; + int gindex; + + if (*e->enumerators != '\0') { + n++; + for (p = e->enumerators; *p != '\0'; p++) + n += (*p == ','); + } + enumerators = PyTuple_New(n); + if (enumerators == NULL) + return NULL; + + enumvalues = PyTuple_New(n); + if (enumvalues == NULL) { + Py_DECREF(enumerators); + return NULL; + } + + p = e->enumerators; + for (i = 0; i < n; i++) { + j = 0; + while (p[j] != ',' && p[j] != '\0') + j++; + tmp = PyString_FromStringAndSize(p, j); + PyTuple_SET_ITEM(enumerators, i, tmp); + + gindex = search_in_globals(&builder->ctx, p, j); + g = &builder->ctx.globals[gindex]; + assert(gindex >= 0 && g->type_op == op); + + tmp = realize_global_int(g); + PyTuple_SET_ITEM(enumvalues, i, tmp); + + p += j + 1; + } + + PyObject *args = NULL; + if (!PyErr_Occurred()) + args = Py_BuildValue("(sOOO)", e->name, enumerators, + enumvalues, basetd); + Py_DECREF(enumerators); + Py_DECREF(enumvalues); if (args == NULL) return NULL; diff --git a/_cffi1/recompiler.py b/_cffi1/recompiler.py --- a/_cffi1/recompiler.py +++ b/_cffi1/recompiler.py @@ -620,10 +620,12 @@ size = "sizeof(%s)" % cname signed = "((%s)-1) <= 0" % cname prim = "_cffi_prim_int(%s, %s)" % (size, signed) + allenums = ",".join(tp.enumerators) else: size = xxxx self._lsts["enum"].append( - ' { "%s", %d, %s },' % (tp.name, type_index, prim)) + ' { "%s", %d, %s,\n "%s" },' % (tp.name, type_index, + prim, allenums)) def _generate_cpy_enum_ctx(self, tp, name): if tp.has_c_name(): diff --git a/_cffi1/test_verify1.py b/_cffi1/test_verify1.py --- a/_cffi1/test_verify1.py +++ b/_cffi1/test_verify1.py @@ -688,6 +688,9 @@ # try again ffi.verify("enum ee { EE1=10, EE2, EE3=-10, EE4 };") assert ffi.string(ffi.cast('enum ee', 11)) == "EE2" + # + assert ffi.typeof("enum ee").relements == {'EE1': 10, 'EE2': 11, 'EE3': -10} + assert ffi.typeof("enum ee").elements == {10: 'EE1', 11: 'EE2', -10: 'EE3'} def test_full_enum(): ffi = FFI() diff --git a/testing/test_verify.py b/testing/test_verify.py --- a/testing/test_verify.py +++ b/testing/test_verify.py @@ -703,6 +703,9 @@ # try again ffi.verify("enum ee { EE1=10, EE2, EE3=-10, EE4 };") assert ffi.string(ffi.cast('enum ee', 11)) == "EE2" + # + assert ffi.typeof("enum ee").relements == {'EE1': 10, 'EE2': 11, 'EE3': -10} + assert ffi.typeof("enum ee").elements == {10: 'EE1', 11: 'EE2', -10: 'EE3'} def test_full_enum(): ffi = FFI() From noreply at buildbot.pypy.org Fri Apr 24 17:49:39 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 24 Apr 2015 17:49:39 +0200 (CEST) Subject: [pypy-commit] pypy.org extradoc: update the values Message-ID: <20150424154939.72A481C01D6@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r597:07be4292a404 Date: 2015-04-24 17:50 +0200 http://bitbucket.org/pypy/pypy.org/changeset/07be4292a404/ Log: update the values diff --git a/don1.html b/don1.html --- a/don1.html +++ b/don1.html @@ -15,7 +15,7 @@ - $59150 of $105000 (56.3%) + $59164 of $105000 (56.3%)

From noreply at buildbot.pypy.org Fri Apr 24 18:01:24 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 24 Apr 2015 18:01:24 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: anonymous enums Message-ID: <20150424160124.75F951C069E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1807:779034948ac1 Date: 2015-04-24 18:01 +0200 http://bitbucket.org/cffi/cffi/changeset/779034948ac1/ Log: anonymous enums diff --git a/_cffi1/recompiler.py b/_cffi1/recompiler.py --- a/_cffi1/recompiler.py +++ b/_cffi1/recompiler.py @@ -616,23 +616,20 @@ ' { "%s", _cffi_const_%s, %s },' % (enumerator, enumerator, type_op)) # - if cname is not None: + if cname is not None and '$' not in cname: size = "sizeof(%s)" % cname signed = "((%s)-1) <= 0" % cname - prim = "_cffi_prim_int(%s, %s)" % (size, signed) - allenums = ",".join(tp.enumerators) else: - size = xxxx + basetp = tp.build_baseinttype(self.ffi, []) + size = self.ffi.sizeof(basetp) + signed = int(int(self.ffi.cast(basetp, -1)) < 0) + allenums = ",".join(tp.enumerators) self._lsts["enum"].append( - ' { "%s", %d, %s,\n "%s" },' % (tp.name, type_index, - prim, allenums)) + ' { "%s", %d, _cffi_prim_int(%s, %s),\n' + ' "%s" },' % (tp.name, type_index, size, signed, allenums)) def _generate_cpy_enum_ctx(self, tp, name): - if tp.has_c_name(): - cname = tp.get_c_name('') - else: - cname = None - self._enum_ctx(tp, cname) + self._enum_ctx(tp, tp._get_c_name()) # ---------- # macros: for now only for integers diff --git a/_cffi1/test_verify1.py b/_cffi1/test_verify1.py --- a/_cffi1/test_verify1.py +++ b/_cffi1/test_verify1.py @@ -697,9 +697,10 @@ ffi.cdef("enum ee { EE1, EE2, EE3 };") ffi.verify("enum ee { EE1, EE2, EE3 };") py.test.raises(VerificationError, ffi.verify, "enum ee { EE1, EE2 };") - e = py.test.raises(VerificationError, ffi.verify, - "enum ee { EE1, EE3, EE2 };") - assert str(e.value) == 'enum ee: EE2 has the real value 2, not 1' + # disabled: for now, we always accept and fix transparently constant values + #e = py.test.raises(VerificationError, ffi.verify, + # "enum ee { EE1, EE3, EE2 };") + #assert str(e.value) == 'enum ee: EE2 has the real value 2, not 1' # extra items cannot be seen and have no bad consequence anyway lib = ffi.verify("enum ee { EE1, EE2, EE3, EE4 };") assert lib.EE3 == 2 From noreply at buildbot.pypy.org Fri Apr 24 18:07:14 2015 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 24 Apr 2015 18:07:14 +0200 (CEST) Subject: [pypy-commit] pypy vmprof2: close to be merged branch Message-ID: <20150424160714.58D761C069E@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof2 Changeset: r76928:bae7ef19ac11 Date: 2015-04-24 18:06 +0200 http://bitbucket.org/pypy/pypy/changeset/bae7ef19ac11/ Log: close to be merged branch From noreply at buildbot.pypy.org Fri Apr 24 18:07:17 2015 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 24 Apr 2015 18:07:17 +0200 (CEST) Subject: [pypy-commit] pypy default: Merge vmprof2 Message-ID: <20150424160717.10C7B1C069E@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r76929:97193bc80d75 Date: 2015-04-24 18:07 +0200 http://bitbucket.org/pypy/pypy/changeset/97193bc80d75/ Log: Merge vmprof2 This branch enables the module _vmprof which is basis for vmprof, a lightweight statistical profiler diff too long, truncating to 2000 out of 4350 lines diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -420,3 +420,10 @@ the terms of the GPL license version 2 or any later version. Thus the gdbm module, provided in the file lib_pypy/gdbm.py, is redistributed under the terms of the GPL license as well. + +License for 'pypy/module/_vmprof/src' +-------------------------------------- + +The code is based on gperftools. You may see a copy of the License for it at + + https://code.google.com/p/gperftools/source/browse/COPYING diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -38,6 +38,9 @@ "_csv", "cppyy", "_pypyjson" ]) +if sys.platform.startswith('linux') and sys.maxint > 2147483647: + working_modules.add('_vmprof') + translation_modules = default_modules.copy() translation_modules.update([ "fcntl", "time", "select", "signal", "_rawffi", "zlib", "struct", "_md5", @@ -99,6 +102,7 @@ "_hashlib" : ["pypy.module._ssl.interp_ssl"], "_minimal_curses": ["pypy.module._minimal_curses.fficurses"], "_continuation": ["rpython.rlib.rstacklet"], + "_vmprof" : ["pypy.module._vmprof.interp_vmprof"], } def get_module_validator(modname): diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -11,7 +11,7 @@ INT_MIN, INT_MAX, UINT_MAX, USHRT_MAX from pypy.interpreter.executioncontext import (ExecutionContext, ActionFlag, - UserDelAction) + UserDelAction, CodeUniqueIds) from pypy.interpreter.error import OperationError, new_exception_class, oefmt from pypy.interpreter.argument import Arguments from pypy.interpreter.miscutils import ThreadLocals, make_weak_value_dictionary @@ -388,6 +388,7 @@ self.actionflag = ActionFlag() # changed by the signal module self.check_signal_action = None # changed by the signal module self.user_del_action = UserDelAction(self) + self.code_unique_ids = CodeUniqueIds() self._code_of_sys_exc_info = None # can be overridden to a subclass @@ -666,6 +667,16 @@ assert ec is not None return ec + def register_code_callback(self, callback): + cui = self.code_unique_ids + cui.code_callback = callback + + def register_code_object(self, pycode): + cui = self.code_unique_ids + if cui.code_callback is None: + return + cui.code_callback(self, pycode) + def _freeze_(self): return True diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -579,3 +579,11 @@ # there is no list of length n: if n is large, then the GC # will run several times while walking the list, but it will # see lower and lower memory usage, with no lower bound of n. + +class CodeUniqueIds(object): + def __init__(self): + if sys.maxint == 2147483647: + self.code_unique_id = 0 # XXX this is wrong, it won't work on 32bit + else: + self.code_unique_id = 0x7000000000000000 + self.code_callback = None diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -14,9 +14,10 @@ CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS, CO_NESTED, CO_GENERATOR, CO_KILL_DOCSTRING, CO_YIELD_INSIDE_TRY) from pypy.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT -from rpython.rlib.rarithmetic import intmask +from rpython.rlib.rarithmetic import intmask, r_longlong from rpython.rlib.objectmodel import compute_hash from rpython.rlib import jit +from rpython.rlib.debug import debug_start, debug_stop, debug_print class BytecodeCorruption(Exception): @@ -54,8 +55,9 @@ "CPython-style code objects." _immutable_ = True _immutable_fields_ = ["co_consts_w[*]", "co_names_w[*]", "co_varnames[*]", - "co_freevars[*]", "co_cellvars[*]", "_args_as_cellvars[*]"] - + "co_freevars[*]", "co_cellvars[*]", + "_args_as_cellvars[*]"] + def __init__(self, space, argcount, nlocals, stacksize, flags, code, consts, names, varnames, filename, name, firstlineno, lnotab, freevars, cellvars, @@ -83,6 +85,7 @@ self.magic = magic self._signature = cpython_code_signature(self) self._initialize() + space.register_code_object(self) def _initialize(self): if self.co_cellvars: @@ -124,6 +127,15 @@ from pypy.objspace.std.mapdict import init_mapdict_cache init_mapdict_cache(self) + cui = self.space.code_unique_ids + self._unique_id = cui.code_unique_id + cui.code_unique_id += 4 # so we have two bits that we can mark stuff + # with + + def _get_full_name(self): + return "py:%s:%d:%s" % (self.co_name, self.co_firstlineno, + self.co_filename) + def _cleanup_(self): if (self.magic == cpython_magic and '__pypy__' not in sys.builtin_module_names): diff --git a/pypy/module/_vmprof/__init__.py b/pypy/module/_vmprof/__init__.py new file mode 100644 --- /dev/null +++ b/pypy/module/_vmprof/__init__.py @@ -0,0 +1,18 @@ +from pypy.interpreter.mixedmodule import MixedModule + +class Module(MixedModule): + """ + Write me :) + """ + appleveldefs = { + } + + interpleveldefs = { + 'enable': 'interp_vmprof.enable', + 'disable': 'interp_vmprof.disable', + } + + def setup_after_space_initialization(self): + # force the __extend__ hacks to occur early + from pypy.module._vmprof.interp_vmprof import VMProf + self.vmprof = VMProf() diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py new file mode 100644 --- /dev/null +++ b/pypy/module/_vmprof/interp_vmprof.py @@ -0,0 +1,240 @@ +import py, os, sys +from rpython.rtyper.lltypesystem import lltype, rffi, llmemory +from rpython.translator.tool.cbuild import ExternalCompilationInfo +from rpython.rtyper.annlowlevel import cast_instance_to_gcref, cast_base_ptr_to_instance +from rpython.rlib.objectmodel import we_are_translated +from rpython.rlib import jit, rposix, rgc +from rpython.rlib.rarithmetic import ovfcheck_float_to_int +from rpython.rtyper.tool import rffi_platform as platform +from rpython.rlib.rstring import StringBuilder +from pypy.interpreter.baseobjspace import W_Root +from pypy.interpreter.error import oefmt, wrap_oserror, OperationError +from pypy.interpreter.gateway import unwrap_spec +from pypy.interpreter.pyframe import PyFrame +from pypy.interpreter.pycode import PyCode + +ROOT = py.path.local(__file__).join('..') +SRC = ROOT.join('src') + +# by default, we statically link vmprof.c into pypy; however, if you set +# DYNAMIC_VMPROF to True, it will be dynamically linked to the libvmprof.so +# which is expected to be inside pypy/module/_vmprof/src: this is very useful +# during development. Note that you have to manually build libvmprof by +# running make inside the src dir +DYNAMIC_VMPROF = False + +eci_kwds = dict( + include_dirs = [SRC], + includes = ['vmprof.h', 'trampoline.h'], + separate_module_files = [SRC.join('trampoline.asmgcc.s')], + link_extra = ['-Wl,-Bstatic', '-lunwind', '-Wl,-Bdynamic', '-llzma'], + + post_include_bits=[""" + void pypy_vmprof_init(void); + """], + + separate_module_sources=[""" + void pypy_vmprof_init(void) { + vmprof_set_mainloop(pypy_execute_frame_trampoline, 0, + NULL); + } + """], + ) + + +if DYNAMIC_VMPROF: + eci_kwds['libraries'] += ['vmprof'] + eci_kwds['link_extra'] = ['-Wl,-rpath,%s' % SRC, '-L%s' % SRC] +else: + eci_kwds['separate_module_files'] += [SRC.join('vmprof.c')] + +eci = ExternalCompilationInfo(**eci_kwds) + +check_eci = eci.merge(ExternalCompilationInfo(separate_module_files=[ + SRC.join('fake_pypy_api.c')])) + +platform.verify_eci(check_eci) + +pypy_execute_frame_trampoline = rffi.llexternal( + "pypy_execute_frame_trampoline", + [llmemory.GCREF, llmemory.GCREF, llmemory.GCREF, lltype.Signed], + llmemory.GCREF, + compilation_info=eci, + _nowrapper=True, sandboxsafe=True, + random_effects_on_gcobjs=True) + +pypy_vmprof_init = rffi.llexternal("pypy_vmprof_init", [], lltype.Void, + compilation_info=eci) +vmprof_enable = rffi.llexternal("vmprof_enable", + [rffi.INT, rffi.LONG, rffi.INT, + rffi.CCHARP, rffi.INT], + rffi.INT, compilation_info=eci, + save_err=rffi.RFFI_SAVE_ERRNO) +vmprof_disable = rffi.llexternal("vmprof_disable", [], rffi.INT, + compilation_info=eci, + save_err=rffi.RFFI_SAVE_ERRNO) + +vmprof_register_virtual_function = rffi.llexternal( + "vmprof_register_virtual_function", + [rffi.CCHARP, rffi.VOIDP, rffi.VOIDP], lltype.Void, + compilation_info=eci, _nowrapper=True) + +original_execute_frame = PyFrame.execute_frame.im_func +original_execute_frame.c_name = 'pypy_pyframe_execute_frame' +original_execute_frame._dont_inline_ = True + +class __extend__(PyFrame): + def execute_frame(frame, w_inputvalue=None, operr=None): + # go through the asm trampoline ONLY if we are translated but not being JITted. + # + # If we are not translated, we obviously don't want to go through the + # trampoline because there is no C function it can call. + # + # If we are being JITted, we want to skip the trampoline, else the JIT + # cannot see throug it + if we_are_translated() and not jit.we_are_jitted(): + # if we are translated, call the trampoline + gc_frame = cast_instance_to_gcref(frame) + gc_inputvalue = cast_instance_to_gcref(w_inputvalue) + gc_operr = cast_instance_to_gcref(operr) + unique_id = frame.pycode._unique_id + gc_result = pypy_execute_frame_trampoline(gc_frame, gc_inputvalue, + gc_operr, unique_id) + return cast_base_ptr_to_instance(W_Root, gc_result) + else: + return original_execute_frame(frame, w_inputvalue, operr) + + + +def write_long_to_string_builder(l, b): + if sys.maxint == 2147483647: + b.append(chr(l & 0xff)) + b.append(chr((l >> 8) & 0xff)) + b.append(chr((l >> 16) & 0xff)) + b.append(chr((l >> 24) & 0xff)) + else: + b.append(chr(l & 0xff)) + b.append(chr((l >> 8) & 0xff)) + b.append(chr((l >> 16) & 0xff)) + b.append(chr((l >> 24) & 0xff)) + b.append(chr((l >> 32) & 0xff)) + b.append(chr((l >> 40) & 0xff)) + b.append(chr((l >> 48) & 0xff)) + b.append(chr((l >> 56) & 0xff)) + +def try_cast_to_pycode(gcref): + return rgc.try_cast_gcref_to_instance(PyCode, gcref) + +MAX_CODES = 1000 + +class VMProf(object): + def __init__(self): + self.is_enabled = False + self.ever_enabled = False + self.fileno = -1 + self.current_codes = [] + + def enable(self, space, fileno, period_usec): + if self.is_enabled: + raise oefmt(space.w_ValueError, "_vmprof already enabled") + self.fileno = fileno + self.is_enabled = True + self.write_header(fileno, period_usec) + if not self.ever_enabled: + if we_are_translated(): + pypy_vmprof_init() + self.ever_enabled = True + self.gather_all_code_objs(space) + space.register_code_callback(vmprof_register_code) + if we_are_translated(): + # does not work untranslated + res = vmprof_enable(fileno, period_usec, 0, + lltype.nullptr(rffi.CCHARP.TO), 0) + else: + res = 0 + if res == -1: + raise wrap_oserror(space, OSError(rposix.get_saved_errno(), + "_vmprof.enable")) + + def gather_all_code_objs(self, space): + all_code_objs = rgc.do_get_objects(try_cast_to_pycode) + for code in all_code_objs: + self.register_code(space, code) + + def write_header(self, fileno, period_usec): + assert period_usec > 0 + b = StringBuilder() + write_long_to_string_builder(0, b) + write_long_to_string_builder(3, b) + write_long_to_string_builder(0, b) + write_long_to_string_builder(period_usec, b) + write_long_to_string_builder(0, b) + b.append('\x04') # interp name + b.append(chr(len('pypy'))) + b.append('pypy') + os.write(fileno, b.build()) + + def register_code(self, space, code): + if self.fileno == -1: + raise OperationError(space.w_RuntimeError, + space.wrap("vmprof not running")) + self.current_codes.append(code) + if len(self.current_codes) >= MAX_CODES: + self._flush_codes(space) + + def _flush_codes(self, space): + b = StringBuilder() + for code in self.current_codes: + name = code._get_full_name() + b.append('\x02') + write_long_to_string_builder(code._unique_id, b) + write_long_to_string_builder(len(name), b) + b.append(name) + os.write(self.fileno, b.build()) + self.current_codes = [] + + def disable(self, space): + if not self.is_enabled: + raise oefmt(space.w_ValueError, "_vmprof not enabled") + self.is_enabled = False + space.register_code_callback(None) + self._flush_codes(space) + self.fileno = -1 + if we_are_translated(): + # does not work untranslated + res = vmprof_disable() + else: + res = 0 + if res == -1: + raise wrap_oserror(space, OSError(rposix.get_saved_errno(), + "_vmprof.disable")) + +def vmprof_register_code(space, code): + from pypy.module._vmprof import Module + mod_vmprof = space.getbuiltinmodule('_vmprof') + assert isinstance(mod_vmprof, Module) + mod_vmprof.vmprof.register_code(space, code) + + at unwrap_spec(fileno=int, period=float) +def enable(space, fileno, period=0.01): # default 100 Hz + from pypy.module._vmprof import Module + mod_vmprof = space.getbuiltinmodule('_vmprof') + assert isinstance(mod_vmprof, Module) + # + try: + period_usec = ovfcheck_float_to_int(period * 1000000.0 + 0.5) + if period_usec <= 0 or period_usec >= 1e6: + # we don't want seconds here at all + raise ValueError + except (ValueError, OverflowError): + raise OperationError(space.w_ValueError, + space.wrap("'period' too large or non positive")) + # + mod_vmprof.vmprof.enable(space, fileno, period_usec) + +def disable(space): + from pypy.module._vmprof import Module + mod_vmprof = space.getbuiltinmodule('_vmprof') + assert isinstance(mod_vmprof, Module) + mod_vmprof.vmprof.disable(space) + diff --git a/pypy/module/_vmprof/src/config.h b/pypy/module/_vmprof/src/config.h new file mode 100644 --- /dev/null +++ b/pypy/module/_vmprof/src/config.h @@ -0,0 +1,2 @@ +#define HAVE_SYS_UCONTEXT_H +#define PC_FROM_UCONTEXT uc_mcontext.gregs[REG_RIP] diff --git a/pypy/module/_vmprof/src/fake_pypy_api.c b/pypy/module/_vmprof/src/fake_pypy_api.c new file mode 100644 --- /dev/null +++ b/pypy/module/_vmprof/src/fake_pypy_api.c @@ -0,0 +1,21 @@ + +long pypy_jit_stack_depth_at_loc(long x) +{ + return 0; +} + +void *pypy_find_codemap_at_addr(long x) +{ + return (void *)0; +} + +long pypy_yield_codemap_at_addr(void *x, long y, long *a) +{ + return 0; +} + +void pypy_pyframe_execute_frame(void) +{ +} + +volatile int pypy_codemap_currently_invalid = 0; diff --git a/pypy/module/_vmprof/src/get_custom_offset.c b/pypy/module/_vmprof/src/get_custom_offset.c new file mode 100644 --- /dev/null +++ b/pypy/module/_vmprof/src/get_custom_offset.c @@ -0,0 +1,66 @@ + +extern volatile int pypy_codemap_currently_invalid; + +void *pypy_find_codemap_at_addr(long addr, long *start_addr); +long pypy_yield_codemap_at_addr(void *codemap_raw, long addr, + long *current_pos_addr); +long pypy_jit_stack_depth_at_loc(long loc); + + +void vmprof_set_tramp_range(void* start, void* end) +{ +} + +int custom_sanity_check() +{ + return !pypy_codemap_currently_invalid; +} + +static ptrdiff_t vmprof_unw_get_custom_offset(void* ip, void *cp) { + intptr_t ip_l = (intptr_t)ip; + return pypy_jit_stack_depth_at_loc(ip_l); +} + +static long vmprof_write_header_for_jit_addr(void **result, long n, + void *ip, int max_depth) +{ + void *codemap; + long current_pos = 0; + intptr_t id; + long start_addr = 0; + intptr_t addr = (intptr_t)ip; + int start, k; + void *tmp; + + codemap = pypy_find_codemap_at_addr(addr, &start_addr); + if (codemap == NULL) + // not a jit code at all + return n; + + // modify the last entry to point to start address and not the random one + // in the middle + result[n - 1] = (void*)start_addr; + result[n] = (void*)2; + n++; + start = n; + while (n < max_depth) { + id = pypy_yield_codemap_at_addr(codemap, addr, ¤t_pos); + if (id == -1) + // finish + break; + if (id == 0) + continue; // not main codemap + result[n++] = (void *)id; + } + k = 0; + while (k < (n - start) / 2) { + tmp = result[start + k]; + result[start + k] = result[n - k - 1]; + result[n - k - 1] = tmp; + k++; + } + if (n < max_depth) { + result[n++] = (void*)3; + } + return n; +} diff --git a/pypy/module/_vmprof/src/getpc.h b/pypy/module/_vmprof/src/getpc.h new file mode 100644 --- /dev/null +++ b/pypy/module/_vmprof/src/getpc.h @@ -0,0 +1,187 @@ +// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// Author: Craig Silverstein +// +// This is an internal header file used by profiler.cc. It defines +// the single (inline) function GetPC. GetPC is used in a signal +// handler to figure out the instruction that was being executed when +// the signal-handler was triggered. +// +// To get this, we use the ucontext_t argument to the signal-handler +// callback, which holds the full context of what was going on when +// the signal triggered. How to get from a ucontext_t to a Program +// Counter is OS-dependent. + +#ifndef BASE_GETPC_H_ +#define BASE_GETPC_H_ + +#include "config.h" + +// On many linux systems, we may need _GNU_SOURCE to get access to +// the defined constants that define the register we want to see (eg +// REG_EIP). Note this #define must come first! +#define _GNU_SOURCE 1 +// If #define _GNU_SOURCE causes problems, this might work instead. +// It will cause problems for FreeBSD though!, because it turns off +// the needed __BSD_VISIBLE. +//#define _XOPEN_SOURCE 500 + +#include // for memcmp +#if defined(HAVE_SYS_UCONTEXT_H) +#include +#elif defined(HAVE_UCONTEXT_H) +#include // for ucontext_t (and also mcontext_t) +#elif defined(HAVE_CYGWIN_SIGNAL_H) +#include +typedef ucontext ucontext_t; +#endif + + +// Take the example where function Foo() calls function Bar(). For +// many architectures, Bar() is responsible for setting up and tearing +// down its own stack frame. In that case, it's possible for the +// interrupt to happen when execution is in Bar(), but the stack frame +// is not properly set up (either before it's done being set up, or +// after it's been torn down but before Bar() returns). In those +// cases, the stack trace cannot see the caller function anymore. +// +// GetPC can try to identify this situation, on architectures where it +// might occur, and unwind the current function call in that case to +// avoid false edges in the profile graph (that is, edges that appear +// to show a call skipping over a function). To do this, we hard-code +// in the asm instructions we might see when setting up or tearing +// down a stack frame. +// +// This is difficult to get right: the instructions depend on the +// processor, the compiler ABI, and even the optimization level. This +// is a best effort patch -- if we fail to detect such a situation, or +// mess up the PC, nothing happens; the returned PC is not used for +// any further processing. +struct CallUnrollInfo { + // Offset from (e)ip register where this instruction sequence + // should be matched. Interpreted as bytes. Offset 0 is the next + // instruction to execute. Be extra careful with negative offsets in + // architectures of variable instruction length (like x86) - it is + // not that easy as taking an offset to step one instruction back! + int pc_offset; + // The actual instruction bytes. Feel free to make it larger if you + // need a longer sequence. + unsigned char ins[16]; + // How many bytes to match from ins array? + int ins_size; + // The offset from the stack pointer (e)sp where to look for the + // call return address. Interpreted as bytes. + int return_sp_offset; +}; + + +// The dereferences needed to get the PC from a struct ucontext were +// determined at configure time, and stored in the macro +// PC_FROM_UCONTEXT in config.h. The only thing we need to do here, +// then, is to do the magic call-unrolling for systems that support it. + +// -- Special case 1: linux x86, for which we have CallUnrollInfo +#if defined(__linux) && defined(__i386) && defined(__GNUC__) +static const CallUnrollInfo callunrollinfo[] = { + // Entry to a function: push %ebp; mov %esp,%ebp + // Top-of-stack contains the caller IP. + { 0, + {0x55, 0x89, 0xe5}, 3, + 0 + }, + // Entry to a function, second instruction: push %ebp; mov %esp,%ebp + // Top-of-stack contains the old frame, caller IP is +4. + { -1, + {0x55, 0x89, 0xe5}, 3, + 4 + }, + // Return from a function: RET. + // Top-of-stack contains the caller IP. + { 0, + {0xc3}, 1, + 0 + } +}; + +inline void* GetPC(ucontext_t *signal_ucontext) { + // See comment above struct CallUnrollInfo. Only try instruction + // flow matching if both eip and esp looks reasonable. + const int eip = signal_ucontext->uc_mcontext.gregs[REG_EIP]; + const int esp = signal_ucontext->uc_mcontext.gregs[REG_ESP]; + if ((eip & 0xffff0000) != 0 && (~eip & 0xffff0000) != 0 && + (esp & 0xffff0000) != 0) { + char* eip_char = reinterpret_cast(eip); + for (int i = 0; i < sizeof(callunrollinfo)/sizeof(*callunrollinfo); ++i) { + if (!memcmp(eip_char + callunrollinfo[i].pc_offset, + callunrollinfo[i].ins, callunrollinfo[i].ins_size)) { + // We have a match. + void **retaddr = (void**)(esp + callunrollinfo[i].return_sp_offset); + return *retaddr; + } + } + } + return (void*)eip; +} + +// Special case #2: Windows, which has to do something totally different. +#elif defined(_WIN32) || defined(__CYGWIN__) || defined(__CYGWIN32__) || defined(__MINGW32__) +// If this is ever implemented, probably the way to do it is to have +// profiler.cc use a high-precision timer via timeSetEvent: +// http://msdn2.microsoft.com/en-us/library/ms712713.aspx +// We'd use it in mode TIME_CALLBACK_FUNCTION/TIME_PERIODIC. +// The callback function would be something like prof_handler, but +// alas the arguments are different: no ucontext_t! I don't know +// how we'd get the PC (using StackWalk64?) +// http://msdn2.microsoft.com/en-us/library/ms680650.aspx + +#include "base/logging.h" // for RAW_LOG +#ifndef HAVE_CYGWIN_SIGNAL_H +typedef int ucontext_t; +#endif + +inline void* GetPC(ucontext_t *signal_ucontext) { + RAW_LOG(ERROR, "GetPC is not yet implemented on Windows\n"); + return NULL; +} + +// Normal cases. If this doesn't compile, it's probably because +// PC_FROM_UCONTEXT is the empty string. You need to figure out +// the right value for your system, and add it to the list in +// configure.ac (or set it manually in your config.h). +#else +inline void* GetPC(ucontext_t *signal_ucontext) { + return (void*)signal_ucontext->PC_FROM_UCONTEXT; // defined in config.h +} + +#endif + +#endif // BASE_GETPC_H_ diff --git a/pypy/module/_vmprof/src/trampoline.asmgcc.s b/pypy/module/_vmprof/src/trampoline.asmgcc.s new file mode 100644 --- /dev/null +++ b/pypy/module/_vmprof/src/trampoline.asmgcc.s @@ -0,0 +1,16 @@ +// NOTE: you need to use TABs, not spaces! + + .text + .p2align 4,,-1 + .globl pypy_execute_frame_trampoline + .type pypy_execute_frame_trampoline, @function +pypy_execute_frame_trampoline: + .cfi_startproc + pushq %rcx + .cfi_def_cfa_offset 16 + call pypy_pyframe_execute_frame at PLT + popq %rcx + .cfi_def_cfa_offset 8 + ret + .cfi_endproc + .size pypy_execute_frame_trampoline, .-pypy_execute_frame_trampoline diff --git a/pypy/module/_vmprof/src/trampoline.h b/pypy/module/_vmprof/src/trampoline.h new file mode 100644 --- /dev/null +++ b/pypy/module/_vmprof/src/trampoline.h @@ -0,0 +1,1 @@ +void* pypy_execute_frame_trampoline(void*, void*, void*, long); diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c new file mode 100644 --- /dev/null +++ b/pypy/module/_vmprof/src/vmprof.c @@ -0,0 +1,398 @@ +/* VMPROF + * + * statistical sampling profiler specifically designed to profile programs + * which run on a Virtual Machine and/or bytecode interpreter, such as Python, + * etc. + * + * The logic to dump the C stack traces is partly stolen from the code in gperftools. + * The file "getpc.h" has been entirely copied from gperftools. + * + * Tested only on gcc, linux, x86_64. + * + * Copyright (C) 2014-2015 + * Antonio Cuni - anto.cuni at gmail.com + * Maciej Fijalkowski - fijall at gmail.com + * + */ + + +#include "getpc.h" // should be first to get the _GNU_SOURCE dfn +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define UNW_LOCAL_ONLY +#include + +#include "vmprof.h" + +#define _unused(x) ((void)x) + +#define MAX_FUNC_NAME 128 +#define MAX_STACK_DEPTH 1024 +#define BUFFER_SIZE 8192 + + +static int profile_file = 0; +static char profile_write_buffer[BUFFER_SIZE]; +static int profile_buffer_position = 0; +void* vmprof_mainloop_func; +static ptrdiff_t mainloop_sp_offset; +static vmprof_get_virtual_ip_t mainloop_get_virtual_ip; +static long last_period_usec = 0; +static int atfork_hook_installed = 0; + + +/* ************************************************************* + * functions to write a profile file compatible with gperftools + * ************************************************************* + */ + +#define MARKER_STACKTRACE '\x01' +#define MARKER_VIRTUAL_IP '\x02' +#define MARKER_TRAILER '\x03' + +static void prof_word(long x) { + ((long*)(profile_write_buffer + profile_buffer_position))[0] = x; + profile_buffer_position += sizeof(long); +} + +static void prof_header(long period_usec) { + // XXX never used here? + prof_word(0); + prof_word(3); + prof_word(0); + prof_word(period_usec); + prof_word(0); + write(profile_file, profile_write_buffer, profile_buffer_position); + profile_buffer_position = 0; +} + +static void prof_write_stacktrace(void** stack, int depth, int count) { + int i; + char marker = MARKER_STACKTRACE; + + profile_write_buffer[profile_buffer_position++] = MARKER_STACKTRACE; + prof_word(count); + prof_word(depth); + for(i=0; isp = bp; + bp -= sizeof(void*); + cp2->ip = ((void**)bp)[0]; + // the ret is on the top of the stack minus WORD + return 1; + } +} + + +/* ************************************************************* + * functions to dump the stack trace + * ************************************************************* + */ + +// The original code here has a comment, "stolen from pprof", +// about a "__thread int recursive". But general __thread +// variables are not really supposed to be accessed from a +// signal handler. Moreover, we are using SIGPROF, which +// should not be recursively called on the same thread. +//static __thread int recursive; + +int get_stack_trace(void** result, int max_depth, ucontext_t *ucontext) { + void *ip; + int n = 0; + unw_cursor_t cursor; + unw_context_t uc = *ucontext; + //if (recursive) { + // return 0; + //} + if (!custom_sanity_check()) { + return 0; + } + //++recursive; + + int ret = unw_init_local(&cursor, &uc); + assert(ret >= 0); + _unused(ret); + int first_run = 1; + + while (n < max_depth) { + if (unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *) &ip) < 0) { + break; + } + + unw_proc_info_t pip; + unw_get_proc_info(&cursor, &pip); + + /* char funcname[4096]; */ + /* unw_word_t offset; */ + /* unw_get_proc_name(&cursor, funcname, 4096, &offset); */ + /* printf("%s+%#lx <%p>\n", funcname, offset, ip); */ + + /* if n==0, it means that the signal handler interrupted us while we + were in the trampoline, so we are not executing (yet) the real main + loop function; just skip it */ + if (vmprof_mainloop_func && + (void*)pip.start_ip == (void*)vmprof_mainloop_func && + n > 0) { + // found main loop stack frame + void* sp; + unw_get_reg(&cursor, UNW_REG_SP, (unw_word_t *) &sp); + void *arg_addr = (char*)sp + mainloop_sp_offset; + void **arg_ptr = (void**)arg_addr; + // fprintf(stderr, "stacktrace mainloop: rsp %p &f2 %p offset %ld\n", + // sp, arg_addr, mainloop_sp_offset); + if (mainloop_get_virtual_ip) { + ip = mainloop_get_virtual_ip(*arg_ptr); + } else { + ip = *arg_ptr; + } + } + + result[n++] = ip; + n = vmprof_write_header_for_jit_addr(result, n, ip, max_depth); + if (vmprof_unw_step(&cursor, first_run) <= 0) { + break; + } + first_run = 0; + } + //--recursive; + return n; +} + + +static int __attribute__((noinline)) frame_forcer(int rv) { + return rv; +} + +static void sigprof_handler(int sig_nr, siginfo_t* info, void *ucontext) { + void* stack[MAX_STACK_DEPTH]; + int saved_errno = errno; + stack[0] = GetPC((ucontext_t*)ucontext); + int depth = frame_forcer(get_stack_trace(stack+1, MAX_STACK_DEPTH-1, ucontext)); + depth++; // To account for pc value in stack[0]; + prof_write_stacktrace(stack, depth, 1); + errno = saved_errno; +} + +/* ************************************************************* + * functions to enable/disable the profiler + * ************************************************************* + */ + +static int open_profile(int fd, long period_usec, int write_header, char *s, + int slen) { + if ((fd = dup(fd)) == -1) { + return -1; + } + profile_buffer_position = 0; + profile_file = fd; + if (write_header) + prof_header(period_usec); + if (s) + write(profile_file, s, slen); + return 0; +} + +static int close_profile(void) { + // XXX all of this can happily fail + FILE* src; + char buf[BUFSIZ]; + size_t size; + int marker = MARKER_TRAILER; + write(profile_file, &marker, 1); + + // copy /proc/PID/maps to the end of the profile file + sprintf(buf, "/proc/%d/maps", getpid()); + src = fopen(buf, "r"); + while ((size = fread(buf, 1, BUFSIZ, src))) { + write(profile_file, buf, size); + } + fclose(src); + close(profile_file); + return 0; +} + + +static int install_sigprof_handler(void) { + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sa.sa_sigaction = sigprof_handler; + sa.sa_flags = SA_RESTART | SA_SIGINFO; + if (sigemptyset(&sa.sa_mask) == -1 || + sigaction(SIGPROF, &sa, NULL) == -1) { + return -1; + } + return 0; +} + +static int remove_sigprof_handler(void) { + sighandler_t res = signal(SIGPROF, SIG_DFL); + if (res == SIG_ERR) { + return -1; + } + return 0; +}; + +static int install_sigprof_timer(long period_usec) { + static struct itimerval timer; + last_period_usec = period_usec; + timer.it_interval.tv_sec = 0; + timer.it_interval.tv_usec = period_usec; + timer.it_value = timer.it_interval; + if (setitimer(ITIMER_PROF, &timer, NULL) != 0) { + return -1; + } + return 0; +} + +static int remove_sigprof_timer(void) { + static struct itimerval timer; + last_period_usec = 0; + timer.it_interval.tv_sec = 0; + timer.it_interval.tv_usec = 0; + timer.it_value.tv_sec = 0; + timer.it_value.tv_usec = 0; + if (setitimer(ITIMER_PROF, &timer, NULL) != 0) { + return -1; + } + return 0; +} + +static void atfork_disable_timer(void) { + remove_sigprof_timer(); +} + +static void atfork_enable_timer(void) { + install_sigprof_timer(last_period_usec); +} + +static int install_pthread_atfork_hooks(void) { + /* this is needed to prevent the problems described there: + - http://code.google.com/p/gperftools/issues/detail?id=278 + - http://lists.debian.org/debian-glibc/2010/03/msg00161.html + + TL;DR: if the RSS of the process is large enough, the clone() syscall + will be interrupted by the SIGPROF before it can complete, then + retried, interrupted again and so on, in an endless loop. The + solution is to disable the timer around the fork, and re-enable it + only inside the parent. + */ + if (atfork_hook_installed) + return 0; + int ret = pthread_atfork(atfork_disable_timer, atfork_enable_timer, NULL); + if (ret != 0) + return -1; + atfork_hook_installed = 1; + return 0; +} + +/* ************************************************************* + * public API + * ************************************************************* + */ + +void vmprof_set_mainloop(void* func, ptrdiff_t sp_offset, + vmprof_get_virtual_ip_t get_virtual_ip) { + mainloop_sp_offset = sp_offset; + mainloop_get_virtual_ip = get_virtual_ip; + vmprof_mainloop_func = func; +} + +int vmprof_enable(int fd, long period_usec, int write_header, char *s, + int slen) +{ + assert(period_usec > 0); + if (open_profile(fd, period_usec, write_header, s, slen) == -1) { + return -1; + } + if (install_sigprof_handler() == -1) { + return -1; + } + if (install_sigprof_timer(period_usec) == -1) { + return -1; + } + if (install_pthread_atfork_hooks() == -1) { + return -1; + } + return 0; +} + +int vmprof_disable(void) { + if (remove_sigprof_timer() == -1) { + return -1; + } + if (remove_sigprof_handler() == -1) { + return -1; + } + if (close_profile() == -1) { + return -1; + } + return 0; +} + +void vmprof_register_virtual_function(const char* name, void* start, void* end) { + // XXX unused by pypy + // for now *end is simply ignored + char buf[1024]; + int lgt = strlen(name) + 2 * sizeof(long) + 1; + + if (lgt > 1024) { + lgt = 1024; + } + buf[0] = MARKER_VIRTUAL_IP; + ((void **)(((void*)buf) + 1))[0] = start; + ((long *)(((void*)buf) + 1 + sizeof(long)))[0] = lgt - 2 * sizeof(long) - 1; + strncpy(buf + 2 * sizeof(long) + 1, name, 1024 - 2 * sizeof(long) - 1); + write(profile_file, buf, lgt); +} diff --git a/pypy/module/_vmprof/src/vmprof.h b/pypy/module/_vmprof/src/vmprof.h new file mode 100644 --- /dev/null +++ b/pypy/module/_vmprof/src/vmprof.h @@ -0,0 +1,22 @@ +#ifndef VMPROF_VMPROF_H_ +#define VMPROF_VMPROF_H_ + +#include + +typedef void* (*vmprof_get_virtual_ip_t)(void*); + +extern void* vmprof_mainloop_func; +void vmprof_set_mainloop(void* func, ptrdiff_t sp_offset, + vmprof_get_virtual_ip_t get_virtual_ip); + +void vmprof_register_virtual_function(const char* name, void* start, void* end); + + +int vmprof_enable(int fd, long period_usec, int write_header, char* vips, + int vips_len); +int vmprof_disable(void); + +// XXX: this should be part of _vmprof (the CPython extension), not vmprof (the library) +void vmprof_set_tramp_range(void* start, void* end); + +#endif diff --git a/pypy/module/_vmprof/test/__init__.py b/pypy/module/_vmprof/test/__init__.py new file mode 100644 diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py new file mode 100644 --- /dev/null +++ b/pypy/module/_vmprof/test/test__vmprof.py @@ -0,0 +1,72 @@ + +import tempfile +from pypy.tool.pytest.objspace import gettestobjspace + +class AppTestVMProf(object): + def setup_class(cls): + cls.space = gettestobjspace(usemodules=['_vmprof', 'struct']) + cls.tmpfile = tempfile.NamedTemporaryFile() + cls.w_tmpfileno = cls.space.wrap(cls.tmpfile.fileno()) + cls.w_tmpfilename = cls.space.wrap(cls.tmpfile.name) + cls.tmpfile2 = tempfile.NamedTemporaryFile() + cls.w_tmpfileno2 = cls.space.wrap(cls.tmpfile2.fileno()) + cls.w_tmpfilename2 = cls.space.wrap(cls.tmpfile2.name) + + def test_import_vmprof(self): + import struct, sys + + WORD = struct.calcsize('l') + + def count(s): + i = 0 + count = 0 + i += 5 * WORD # header + assert s[i] == '\x04' + i += 1 # marker + assert s[i] == '\x04' + i += 1 # length + i += len('pypy') + while i < len(s): + if s[i] == '\x03': + break + if s[i] == '\x01': + xxx + assert s[i] == '\x02' + i += 1 + _, size = struct.unpack("ll", s[i:i + 2 * WORD]) + count += 1 + i += 2 * WORD + size + return count + + import _vmprof + _vmprof.enable(self.tmpfileno) + _vmprof.disable() + s = open(self.tmpfilename).read() + no_of_codes = count(s) + assert no_of_codes > 10 + d = {} + + exec """def foo(): + pass + """ in d + + _vmprof.enable(self.tmpfileno2) + + exec """def foo2(): + pass + """ in d + + _vmprof.disable() + s = open(self.tmpfilename2).read() + no_of_codes2 = count(s) + assert "py:foo:" in s + assert "py:foo2:" in s + assert no_of_codes2 >= no_of_codes + 2 # some extra codes from tests + + def test_enable_ovf(self): + import _vmprof + raises(ValueError, _vmprof.enable, 999, 0) + raises(ValueError, _vmprof.enable, 999, -2.5) + raises(ValueError, _vmprof.enable, 999, 1e300) + raises(ValueError, _vmprof.enable, 999, 1e300 * 1e300) + raises(ValueError, _vmprof.enable, 999, (1e300*1e300) / (1e300*1e300)) diff --git a/pypy/module/_vmprof/test/test_direct.py b/pypy/module/_vmprof/test/test_direct.py new file mode 100644 --- /dev/null +++ b/pypy/module/_vmprof/test/test_direct.py @@ -0,0 +1,67 @@ + +import cffi, py + +srcdir = py.path.local(__file__).join("..", "..", "src") + +ffi = cffi.FFI() +ffi.cdef(""" +long vmprof_write_header_for_jit_addr(void **, long, void*, int); +void *pypy_find_codemap_at_addr(long addr, long *start_addr); +long pypy_yield_codemap_at_addr(void *codemap_raw, long addr, + long *current_pos_addr); +long buffer[]; +""") + +lib = ffi.verify(""" +volatile int pypy_codemap_currently_invalid = 0; + +long buffer[] = {0, 0, 0, 0, 0}; + + + +void *pypy_find_codemap_at_addr(long addr, long *start_addr) +{ + return (void*)buffer; +} + +long pypy_yield_codemap_at_addr(void *codemap_raw, long addr, + long *current_pos_addr) +{ + long c = *current_pos_addr; + if (c >= 5) + return -1; + *current_pos_addr = c + 1; + return *((long*)codemap_raw + c); +} + + +""" + open(str(srcdir.join("get_custom_offset.c"))).read()) + +class TestDirect(object): + def test_infrastructure(self): + cont = ffi.new("long[1]", [0]) + buf = lib.pypy_find_codemap_at_addr(0, cont) + assert buf + cont[0] = 0 + next_addr = lib.pypy_yield_codemap_at_addr(buf, 0, cont) + assert cont[0] == 1 + assert not next_addr + lib.buffer[0] = 13 + cont[0] = 0 + next_addr = lib.pypy_yield_codemap_at_addr(buf, 0, cont) + assert int(ffi.cast("long", next_addr)) == 13 + + def test_write_header_for_jit_addr(self): + lib.buffer[0] = 4 + lib.buffer[1] = 8 + lib.buffer[2] = 12 + lib.buffer[3] = 16 + lib.buffer[4] = 0 + buf = ffi.new("long[10]", [0] * 10) + result = ffi.cast("void**", buf) + res = lib.vmprof_write_header_for_jit_addr(result, 0, ffi.NULL, 100) + assert res == 6 + assert buf[0] == 2 + assert buf[1] == 16 + assert buf[2] == 12 + assert buf[3] == 8 diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py --- a/pypy/module/gc/referents.py +++ b/pypy/module/gc/referents.py @@ -44,30 +44,6 @@ return OperationError(space.w_NotImplementedError, space.wrap("operation not implemented by this GC")) -# ____________________________________________________________ - -def clear_gcflag_extra(fromlist): - pending = fromlist[:] - while pending: - gcref = pending.pop() - if rgc.get_gcflag_extra(gcref): - rgc.toggle_gcflag_extra(gcref) - pending.extend(rgc.get_rpy_referents(gcref)) - -def do_get_objects(): - roots = [gcref for gcref in rgc.get_rpy_roots() if gcref] - pending = roots[:] - result_w = [] - while pending: - gcref = pending.pop() - if not rgc.get_gcflag_extra(gcref): - rgc.toggle_gcflag_extra(gcref) - w_obj = try_cast_gcref_to_w_root(gcref) - if w_obj is not None: - result_w.append(w_obj) - pending.extend(rgc.get_rpy_referents(gcref)) - clear_gcflag_extra(roots) - return result_w # ____________________________________________________________ @@ -116,8 +92,8 @@ break # done. Clear flags carefully rgc.toggle_gcflag_extra(gcarg) - clear_gcflag_extra(roots) - clear_gcflag_extra([gcarg]) + rgc.clear_gcflag_extra(roots) + rgc.clear_gcflag_extra([gcarg]) return result_w # ____________________________________________________________ @@ -189,8 +165,7 @@ """Return a list of all app-level objects.""" if not rgc.has_gcflag_extra(): raise missing_operation(space) - result_w = do_get_objects() - rgc.assert_no_more_gcflags() + result_w = rgc.do_get_objects(try_cast_gcref_to_w_root) return space.newlist(result_w) def get_referents(space, args_w): diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py --- a/pypy/module/pypyjit/interp_jit.py +++ b/pypy/module/pypyjit/interp_jit.py @@ -35,6 +35,9 @@ name = opcode_method_names[ord(bytecode.co_code[next_instr])] return '%s #%d %s' % (bytecode.get_repr(), next_instr, name) +def get_unique_id(next_instr, is_being_profiled, bytecode): + return bytecode._unique_id + def should_unroll_one_iteration(next_instr, is_being_profiled, bytecode): return (bytecode.co_flags & CO_GENERATOR) != 0 @@ -45,6 +48,7 @@ virtualizables = ['frame'] pypyjitdriver = PyPyJitDriver(get_printable_location = get_printable_location, + get_unique_id = get_unique_id, should_unroll_one_iteration = should_unroll_one_iteration, name='pypyjit') diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py --- a/pypy/module/pypyjit/test_pypy_c/model.py +++ b/pypy/module/pypyjit/test_pypy_c/model.py @@ -134,7 +134,8 @@ def _ops_for_chunk(self, chunk, include_guard_not_invalidated): for op in chunk.operations: - if op.name != 'debug_merge_point' and \ + if op.name not in ('debug_merge_point', 'enter_portal_frame', + 'leave_portal_frame') and \ (op.name != 'guard_not_invalidated' or include_guard_not_invalidated): yield op diff --git a/rpython/bin/rpython-vmprof b/rpython/bin/rpython-vmprof new file mode 100755 --- /dev/null +++ b/rpython/bin/rpython-vmprof @@ -0,0 +1,28 @@ +#!/usr/bin/env pypy + +"""RPython translation usage: + +rpython target + +run with --help for more information +""" + +import sys, os +sys.path.insert(0, os.path.dirname(os.path.dirname( + os.path.dirname(os.path.realpath(__file__))))) +from rpython.translator.goal.translate import main + +# no implicit targets +if len(sys.argv) == 1: + print __doc__ + sys.exit(1) + +import _vmprof, subprocess +x = subprocess.Popen('gzip > vmprof.log.gz', shell=True, stdin=subprocess.PIPE) +_vmprof.enable(x.stdin.fileno(), 0.001) +try: + main() +finally: + _vmprof.disable() + x.stdin.close() + x.wait() diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py --- a/rpython/jit/backend/arm/assembler.py +++ b/rpython/jit/backend/arm/assembler.py @@ -57,6 +57,7 @@ BaseAssembler.setup_once(self) def setup(self, looptoken): + BaseAssembler.setup(self, looptoken) assert self.memcpy_addr != 0, 'setup_once() not called?' if we_are_translated(): self.debug = False @@ -71,7 +72,6 @@ self.mc.datablockwrapper = self.datablockwrapper self.target_tokens_currently_compiling = {} self.frame_depth_to_patch = [] - self._finish_gcmap = lltype.nullptr(jitframe.GCMAP) def teardown(self): self.current_clt = None @@ -102,7 +102,7 @@ self.store_reg(mc, r.r0, r.fp, ofs) mc.MOV_rr(r.r0.value, r.fp.value) self.gen_func_epilog(mc) - rawstart = mc.materialize(self.cpu.asmmemmgr, []) + rawstart = mc.materialize(self.cpu, []) self.propagate_exception_path = rawstart def _store_and_reset_exception(self, mc, excvalloc=None, exctploc=None, @@ -198,7 +198,7 @@ mc.ADD_ri(r.sp.value, r.sp.value, (len(r.argument_regs) + 2) * WORD) mc.B(self.propagate_exception_path) # - rawstart = mc.materialize(self.cpu.asmmemmgr, []) + rawstart = mc.materialize(self.cpu, []) self.stack_check_slowpath = rawstart def _build_wb_slowpath(self, withcards, withfloats=False, for_frame=False): @@ -255,7 +255,7 @@ # mc.POP([r.ip.value, r.pc.value]) # - rawstart = mc.materialize(self.cpu.asmmemmgr, []) + rawstart = mc.materialize(self.cpu, []) if for_frame: self.wb_slowpath[4] = rawstart else: @@ -276,7 +276,7 @@ callee_only) # return mc.POP([r.ip.value, r.pc.value]) - return mc.materialize(self.cpu.asmmemmgr, []) + return mc.materialize(self.cpu, []) def _build_malloc_slowpath(self, kind): """ While arriving on slowpath, we have a gcpattern on stack 0. @@ -352,7 +352,7 @@ mc.POP([r.ip.value, r.pc.value]) # - rawstart = mc.materialize(self.cpu.asmmemmgr, []) + rawstart = mc.materialize(self.cpu, []) return rawstart def _reload_frame_if_necessary(self, mc): @@ -473,7 +473,7 @@ mc.MOV_rr(r.r0.value, r.fp.value) # self.gen_func_epilog(mc) - rawstart = mc.materialize(self.cpu.asmmemmgr, []) + rawstart = mc.materialize(self.cpu, []) self.failure_recovery_code[exc + 2 * withfloats] = rawstart def generate_quick_failure(self, guardtok): @@ -575,8 +575,8 @@ self.mc.BL(self.stack_check_slowpath, c=c.HI) # call if ip > lr # cpu interface - def assemble_loop(self, logger, loopname, inputargs, operations, looptoken, - log): + def assemble_loop(self, jd_id, unique_id, logger, loopname, inputargs, + operations, looptoken, log): clt = CompiledLoopToken(self.cpu, looptoken.number) looptoken.compiled_loop_token = clt clt._debug_nbargs = len(inputargs) @@ -586,6 +586,9 @@ assert len(set(inputargs)) == len(inputargs) self.setup(looptoken) + #self.codemap_builder.enter_portal_frame(jd_id, unique_id, + # self.mc.get_relative_pos()) + frame_info = self.datablockwrapper.malloc_aligned( jitframe.JITFRAMEINFO_SIZE, alignment=WORD) @@ -659,6 +662,7 @@ assert len(set(inputargs)) == len(inputargs) self.setup(original_loop_token) + #self.codemap.inherit_code_from_position(faildescr.adr_jump_offset) descr_number = compute_unique_id(faildescr) if log: operations = self._inject_debugging_code(faildescr, operations, @@ -850,7 +854,7 @@ # restore registers self._pop_all_regs_from_jitframe(mc, [], self.cpu.supports_floats) mc.POP([r.ip.value, r.pc.value]) # return - self._frame_realloc_slowpath = mc.materialize(self.cpu.asmmemmgr, []) + self._frame_realloc_slowpath = mc.materialize(self.cpu, []) def _load_shadowstack_top(self, mc, reg, gcrootmap): rst = gcrootmap.get_root_stack_top_addr() @@ -879,8 +883,12 @@ self.datablockwrapper.done() # finish using cpu.asmmemmgr self.datablockwrapper = None allblocks = self.get_asmmemmgr_blocks(looptoken) - return self.mc.materialize(self.cpu.asmmemmgr, allblocks, + size = self.mc.get_relative_pos() + res = self.mc.materialize(self.cpu, allblocks, self.cpu.gc_ll_descr.gcrootmap) + #self.cpu.codemap.register_codemap( + # self.codemap.get_final_bytecode(res, size)) + return res def update_frame_depth(self, frame_depth): baseofs = self.cpu.get_baseofs_of_frame_field() diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -373,6 +373,12 @@ return gcmap # ------------------------------------------------------------ + def perform_enter_portal_frame(self, op): + self.assembler.enter_portal_frame(op) + + def perform_leave_portal_frame(self, op): + self.assembler.leave_portal_frame(op) + def perform_extra(self, op, args, fcond): return self.assembler.regalloc_emit_extra(op, args, fcond, self) diff --git a/rpython/jit/backend/arm/runner.py b/rpython/jit/backend/arm/runner.py --- a/rpython/jit/backend/arm/runner.py +++ b/rpython/jit/backend/arm/runner.py @@ -50,16 +50,12 @@ def setup_once(self): self.cpuinfo.arch_version = detect_arch_version() self.cpuinfo.hf_abi = detect_hardfloat() + #self.codemap.setup() self.assembler.setup_once() def finish_once(self): self.assembler.finish_once() - def compile_loop(self, inputargs, operations, looptoken, - log=True, name='', logger=None): - return self.assembler.assemble_loop(logger, name, inputargs, operations, - looptoken, log=log) - def compile_bridge(self, faildescr, inputargs, operations, original_loop_token, log=True, logger=None): clt = original_loop_token.compiled_loop_token diff --git a/rpython/jit/backend/arm/test/support.py b/rpython/jit/backend/arm/test/support.py --- a/rpython/jit/backend/arm/test/support.py +++ b/rpython/jit/backend/arm/test/support.py @@ -24,7 +24,7 @@ def run_asm(asm): BOOTSTRAP_TP = lltype.FuncType([], lltype.Signed) - addr = asm.mc.materialize(asm.cpu.asmmemmgr, [], None) + addr = asm.mc.materialize(asm.cpu, [], None) assert addr % 8 == 0 func = rffi.cast(lltype.Ptr(BOOTSTRAP_TP), addr) asm.mc._dump_trace(addr, 'test.asm') diff --git a/rpython/jit/backend/arm/test/test_calling_convention.py b/rpython/jit/backend/arm/test/test_calling_convention.py --- a/rpython/jit/backend/arm/test/test_calling_convention.py +++ b/rpython/jit/backend/arm/test/test_calling_convention.py @@ -29,7 +29,7 @@ mc = InstrBuilder() mc.MOV_rr(r.r0.value, r.sp.value) mc.MOV_rr(r.pc.value, r.lr.value) - return mc.materialize(self.cpu.asmmemmgr, []) + return mc.materialize(self.cpu, []) def get_alignment_requirements(self): return 8 diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -245,8 +245,8 @@ self.stats = stats or MiniStats() self.vinfo_for_tests = kwds.get('vinfo_for_tests', None) - def compile_loop(self, inputargs, operations, looptoken, log=True, - name='', logger=None): + def compile_loop(self, inputargs, operations, looptoken, jd_id=0, + unique_id=0, log=True, name='', logger=None): clt = model.CompiledLoopToken(self, looptoken.number) looptoken.compiled_loop_token = clt lltrace = LLTrace(inputargs, operations) @@ -1025,6 +1025,12 @@ else: stats.add_merge_point_location(args[1:]) + def execute_enter_portal_frame(self, descr, *args): + pass + + def execute_leave_portal_frame(self, descr, *args): + pass + def execute_new_with_vtable(self, _, vtable): descr = heaptracker.vtable2descr(self.cpu, vtable) return self.cpu.bh_new_with_vtable(vtable, descr) diff --git a/rpython/jit/backend/llsupport/asmmemmgr.py b/rpython/jit/backend/llsupport/asmmemmgr.py --- a/rpython/jit/backend/llsupport/asmmemmgr.py +++ b/rpython/jit/backend/llsupport/asmmemmgr.py @@ -4,7 +4,7 @@ from rpython.rlib import rmmap from rpython.rlib.debug import debug_start, debug_print, debug_stop from rpython.rlib.debug import have_debug_prints -from rpython.rtyper.lltypesystem import lltype, llmemory, rffi +from rpython.rtyper.lltypesystem import lltype, rffi class AsmMemoryManager(object): @@ -212,6 +212,9 @@ gcroot_markers = None + frame_positions = None + frame_assignments = None + def __init__(self, translated=None): if translated is None: translated = we_are_translated() @@ -303,11 +306,11 @@ # debug_stop(logname) - def materialize(self, asmmemmgr, allblocks, gcrootmap=None): + def materialize(self, cpu, allblocks, gcrootmap=None): size = self.get_relative_pos() align = self.ALIGN_MATERIALIZE size += align - 1 - malloced = asmmemmgr.malloc(size, size) + malloced = cpu.asmmemmgr.malloc(size, size) allblocks.append(malloced) rawstart = malloced[0] rawstart = (rawstart + align - 1) & (-align) @@ -316,6 +319,12 @@ assert gcrootmap is not None for pos, mark in self.gcroot_markers: gcrootmap.register_asm_addr(rawstart + pos, mark) + if cpu.HAS_CODEMAP: + cpu.codemap.register_frame_depth_map(rawstart, rawstart + size, + self.frame_positions, + self.frame_assignments) + self.frame_positions = None + self.frame_assignments = None return rawstart def _become_a_plain_block_builder(self): diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py --- a/rpython/jit/backend/llsupport/assembler.py +++ b/rpython/jit/backend/llsupport/assembler.py @@ -1,6 +1,7 @@ from rpython.jit.backend.llsupport import jitframe from rpython.jit.backend.llsupport.memcpy import memcpy_fn, memset_fn from rpython.jit.backend.llsupport.symbolic import WORD +from rpython.jit.backend.llsupport.codemap import CodemapBuilder from rpython.jit.metainterp.history import (INT, REF, FLOAT, JitCellToken, ConstInt, BoxInt, AbstractFailDescr) from rpython.jit.metainterp.resoperation import ResOperation, rop @@ -128,6 +129,11 @@ track_allocation=False) self.gcmap_for_finish[0] = r_uint(1) + def setup(self, looptoken): + if self.cpu.HAS_CODEMAP: + self.codemap_builder = CodemapBuilder() + self._finish_gcmap = lltype.nullptr(jitframe.GCMAP) + def set_debug(self, v): r = self._debug self._debug = v @@ -194,6 +200,17 @@ guardtok.faildescr.rd_locs = positions return fail_descr, target + def enter_portal_frame(self, op): + if self.cpu.HAS_CODEMAP: + self.codemap_builder.enter_portal_frame(op.getarg(0).getint(), + op.getarg(1).getint(), + self.mc.get_relative_pos()) + + def leave_portal_frame(self, op): + if self.cpu.HAS_CODEMAP: + self.codemap_builder.leave_portal_frame(op.getarg(0).getint(), + self.mc.get_relative_pos()) + def call_assembler(self, op, guard_op, argloc, vloc, result_loc, tmploc): self._store_force_index(guard_op) descr = op.getdescr() @@ -276,6 +293,9 @@ # YYY very minor leak -- we need the counters to stay alive # forever, just because we want to report them at the end # of the process + + # XXX the numbers here are ALMOST unique, but not quite, use a counter + # or something struct = lltype.malloc(DEBUG_COUNTER, flavor='raw', track_allocation=False) struct.i = 0 diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py new file mode 100644 --- /dev/null +++ b/rpython/jit/backend/llsupport/codemap.py @@ -0,0 +1,180 @@ + +""" Bytecode for storage in asmmemmgr.jit_codemap. Format is as follows: + + list of tuples of shape (addr, machine code size, bytecode info) + where bytecode info is a string made up of: + 8 bytes unique_id, 4 bytes start_addr (relative), 4 bytes size (relative), + 2 bytes how many items to skip to go to the next on similar level + [so far represented by a list of integers for simplicity] + +""" + +import os +from rpython.rlib import rgc +from rpython.rlib.objectmodel import specialize, we_are_translated +from rpython.rlib.entrypoint import jit_entrypoint +from rpython.rlib.rbisect import bisect_right, bisect_right_addr +from rpython.rlib.rbisect import bisect_left, bisect_left_addr +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.translator.tool.cbuild import ExternalCompilationInfo +from rpython.translator import cdir + + +INT_LIST_PTR = rffi.CArrayPtr(lltype.Signed) + + +srcdir = os.path.join(os.path.dirname(__file__), 'src') + +eci = ExternalCompilationInfo(post_include_bits=[""" +RPY_EXTERN long pypy_jit_codemap_add(unsigned long addr, + unsigned int machine_code_size, + long *bytecode_info, + unsigned int bytecode_info_size); +RPY_EXTERN long *pypy_jit_codemap_del(unsigned long addr, unsigned int size); +RPY_EXTERN unsigned long pypy_jit_codemap_firstkey(void); +RPY_EXTERN void *pypy_find_codemap_at_addr(long addr, long* start_addr); +RPY_EXTERN long pypy_yield_codemap_at_addr(void *codemap_raw, long addr, + long *current_pos_addr); + +RPY_EXTERN long pypy_jit_depthmap_add(unsigned long addr, unsigned int size, + unsigned int stackdepth); +RPY_EXTERN void pypy_jit_depthmap_clear(unsigned long addr, unsigned int size); + +"""], separate_module_sources=[ + open(os.path.join(srcdir, 'skiplist.c'), 'r').read() + + open(os.path.join(srcdir, 'codemap.c'), 'r').read() +], include_dirs=[cdir]) + +def llexternal(name, args, res): + return rffi.llexternal(name, args, res, compilation_info=eci, + releasegil=False) + +pypy_jit_codemap_add = llexternal('pypy_jit_codemap_add', + [lltype.Signed, lltype.Signed, + INT_LIST_PTR, lltype.Signed], + lltype.Signed) +pypy_jit_codemap_del = llexternal('pypy_jit_codemap_del', + [lltype.Signed, lltype.Signed], INT_LIST_PTR) +pypy_jit_codemap_firstkey = llexternal('pypy_jit_codemap_firstkey', + [], lltype.Signed) + +pypy_jit_depthmap_add = llexternal('pypy_jit_depthmap_add', + [lltype.Signed, lltype.Signed, + lltype.Signed], lltype.Signed) +pypy_jit_depthmap_clear = llexternal('pypy_jit_depthmap_clear', + [lltype.Signed, lltype.Signed], + lltype.Void) + +stack_depth_at_loc = llexternal('pypy_jit_stack_depth_at_loc', + [lltype.Signed], lltype.Signed) +find_codemap_at_addr = llexternal('pypy_find_codemap_at_addr', + [lltype.Signed, rffi.CArrayPtr(lltype.Signed)], lltype.Signed) +yield_bytecode_at_addr = llexternal('pypy_yield_codemap_at_addr', + [lltype.Signed, lltype.Signed, + rffi.CArrayPtr(lltype.Signed)], + lltype.Signed) + + +class CodemapStorage(object): + """ An immortal wrapper around underlaying jit codemap data + """ + def setup(self): + if not we_are_translated(): + # in case someone failed to call free(), in tests only anyway + self.free() + + def free(self): + while True: + key = pypy_jit_codemap_firstkey() + if not key: + break + items = pypy_jit_codemap_del(key, 1) + lltype.free(items, flavor='raw', track_allocation=False) + + def free_asm_block(self, start, stop): + items = pypy_jit_codemap_del(start, stop - start) + if items: + lltype.free(items, flavor='raw', track_allocation=False) + pypy_jit_depthmap_clear(start, stop - start) + + def register_frame_depth_map(self, rawstart, rawstop, frame_positions, + frame_assignments): + if not frame_positions: + return + assert len(frame_positions) == len(frame_assignments) + for i in range(len(frame_positions)-1, -1, -1): + pos = rawstart + frame_positions[i] + length = rawstop - pos + if length > 0: + #print "ADD:", pos, length, frame_assignments[i] + pypy_jit_depthmap_add(pos, length, frame_assignments[i]) + rawstop = pos + + def register_codemap(self, (start, size, l)): + items = lltype.malloc(INT_LIST_PTR.TO, len(l), flavor='raw', + track_allocation=False) + for i in range(len(l)): + items[i] = l[i] + if pypy_jit_codemap_add(start, size, items, len(l)) < 0: + lltype.free(items, flavor='raw', track_allocation=False) + + def finish_once(self): + self.free() + +def unpack_traceback(addr): + codemap_raw = find_codemap_at_addr(addr, + lltype.nullptr(rffi.CArray(lltype.Signed))) + if not codemap_raw: + return [] # no codemap for that position + storage = lltype.malloc(rffi.CArray(lltype.Signed), 1, flavor='raw') + storage[0] = 0 + res = [] + while True: + item = yield_bytecode_at_addr(codemap_raw, addr, storage) + if item == -1: + break + res.append(item) + lltype.free(storage, flavor='raw') + return res + + +class CodemapBuilder(object): + def __init__(self): + self.l = [] + self.patch_position = [] + + def enter_portal_frame(self, jd_id, unique_id, relpos): + self.l.append(unique_id) + self.l.append(relpos) + self.patch_position.append(len(self.l)) + self.l.append(0) # marker + self.l.append(0) # second marker + + def leave_portal_frame(self, jd_id, relpos): + to_patch = self.patch_position.pop() + self.l[to_patch] = relpos + self.l[to_patch + 1] = len(self.l) + + def inherit_code_from_position(self, pos): + lst = unpack_traceback(pos) + for item in lst: + self.l.append(item) + self.l.append(0) + self.patch_position.append(len(self.l)) + self.l.append(0) # marker + self.l.append(0) # second marker + + def get_final_bytecode(self, addr, size): + while self.patch_position: + pos = self.patch_position.pop() + self.l[pos] = size + self.l[pos + 1] = len(self.l) + # at the end there should be no zeros, except unique_id which can + # be zero + for i in range(len(self.l) / 4): + item = self.l[i * 4] # unique_id + item = self.l[i * 4 + 2] # end in asm + assert item > 0 + item = self.l[i * 4 + 3] # end in l + assert item > 0 + return (addr, size, self.l) # XXX compact self.l diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -16,13 +16,15 @@ FieldDescr, ArrayDescr, CallDescr, InteriorFieldDescr, FLAG_POINTER, FLAG_FLOAT) from rpython.jit.backend.llsupport.memcpy import memset_fn -from rpython.jit.backend.llsupport.asmmemmgr import AsmMemoryManager +from rpython.jit.backend.llsupport import asmmemmgr, codemap from rpython.rlib.unroll import unrolling_iterable class AbstractLLCPU(AbstractCPU): from rpython.jit.metainterp.typesystem import llhelper as ts + HAS_CODEMAP = False + def __init__(self, rtyper, stats, opts, translate_support_code=False, gcdescr=None): assert type(opts) is not bool @@ -48,7 +50,9 @@ self._setup_exception_handling_translated() else: self._setup_exception_handling_untranslated() - self.asmmemmgr = AsmMemoryManager() + self.asmmemmgr = asmmemmgr.AsmMemoryManager() + if self.HAS_CODEMAP: + self.codemap = codemap.CodemapStorage() self._setup_frame_realloc(translate_support_code) ad = self.gc_ll_descr.getframedescrs(self).arraydescr self.signedarraydescr = ad @@ -78,6 +82,16 @@ def setup(self): pass + def finish_once(self): + if self.HAS_CODEMAP: + self.codemap.finish_once() + + def compile_loop(self, inputargs, operations, looptoken, jd_id=0, + unique_id=0, log=True, name='', logger=None): + return self.assembler.assemble_loop(jd_id, unique_id, logger, name, + inputargs, operations, + looptoken, log=log) + def _setup_frame_realloc(self, translate_support_code): FUNC_TP = lltype.Ptr(lltype.FuncType([llmemory.GCREF, lltype.Signed], llmemory.GCREF)) @@ -212,6 +226,8 @@ for rawstart, rawstop in blocks: self.gc_ll_descr.freeing_block(rawstart, rawstop) self.asmmemmgr.free(rawstart, rawstop) + if self.HAS_CODEMAP: + self.codemap.free_asm_block(rawstart, rawstop) def force(self, addr_of_force_token): frame = rffi.cast(jitframe.JITFRAMEPTR, addr_of_force_token) diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -17,8 +17,6 @@ class GcRewriterAssembler(object): """ This class performs the following rewrites on the list of operations: - - Remove the DEBUG_MERGE_POINTs. - - Turn all NEW_xxx to either a CALL_MALLOC_GC, or a CALL_MALLOC_NURSERY followed by SETFIELDs in order to initialize their GC fields. The two advantages of CALL_MALLOC_NURSERY is that it inlines the common @@ -75,6 +73,8 @@ From noreply at buildbot.pypy.org Sat Apr 25 04:33:43 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Sat, 25 Apr 2015 04:33:43 +0200 (CEST) Subject: [pypy-commit] pypy can_cast: create casting_table and use it in can_cast() Message-ID: <20150425023343.5952F1C0D78@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: can_cast Changeset: r76930:2c0b8658d9ed Date: 2015-04-25 03:33 +0100 http://bitbucket.org/pypy/pypy/changeset/2c0b8658d9ed/ Log: create casting_table and use it in can_cast() diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py --- a/pypy/module/micronumpy/descriptor.py +++ b/pypy/module/micronumpy/descriptor.py @@ -94,7 +94,7 @@ return self.itemtype.box_complex(real, imag) def can_cast_to(self, other): - return True + return self.itemtype.can_cast_to(other.itemtype) def coerce(self, space, w_item): return self.itemtype.coerce(space, self, w_item) diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -129,6 +129,13 @@ else: return alloc_raw_storage(size, track_allocation=False, zero=False) + @classmethod + def basesize(cls): + return rffi.sizeof(cls.T) + + def can_cast_to(self, other): + return casting_table[self.num][other.num] + class Primitive(object): _mixin_ = True @@ -412,6 +419,7 @@ class Integer(Primitive): _mixin_ = True + signed = True def _base_coerce(self, space, w_item): if w_item is None: @@ -568,6 +576,7 @@ char = NPY.UBYTELTR BoxType = boxes.W_UInt8Box format_code = "B" + signed = False class Int16(BaseType, Integer): T = rffi.SHORT @@ -584,6 +593,7 @@ char = NPY.USHORTLTR BoxType = boxes.W_UInt16Box format_code = "H" + signed = False class Int32(BaseType, Integer): T = rffi.INT @@ -600,6 +610,7 @@ char = NPY.UINTLTR BoxType = boxes.W_UInt32Box format_code = "I" + signed = False def _int64_coerce(self, space, w_item): try: @@ -645,6 +656,7 @@ char = NPY.ULONGLONGLTR BoxType = boxes.W_UInt64Box format_code = "Q" + signed = False _coerce = func_with_new_name(_uint64_coerce, '_coerce') @@ -676,6 +688,7 @@ char = NPY.ULONGLTR BoxType = boxes.W_ULongBox format_code = "L" + signed = False _coerce = func_with_new_name(_ulong_coerce, '_coerce') @@ -2392,8 +2405,11 @@ del tp all_float_types = [] +float_types = [] all_int_types = [] +int_types = [] all_complex_types = [] +complex_types = [] def _setup(): # compute alignment @@ -2402,9 +2418,57 @@ tp.alignment = widen(clibffi.cast_type_to_ffitype(tp.T).c_alignment) if issubclass(tp, Float): all_float_types.append((tp, 'float')) + float_types.append(tp) if issubclass(tp, Integer): all_int_types.append((tp, 'int')) + int_types.append(tp) if issubclass(tp, ComplexFloating): all_complex_types.append((tp, 'complex')) + complex_types.append(tp) _setup() del _setup + +casting_table = [[False] * NPY.NTYPES for _ in range(NPY.NTYPES)] +number_types = int_types + float_types + complex_types +all_types = number_types + [ObjectType, StringType, UnicodeType, VoidType] + +def enable_cast(type1, type2): + casting_table[type1.num][type2.num] = True + +for tp in all_types: + enable_cast(tp, tp) + if tp.num != NPY.DATETIME: + enable_cast(Bool, tp) + enable_cast(tp, ObjectType) + enable_cast(tp, VoidType) +enable_cast(StringType, UnicodeType) +#enable_cast(Bool, TimeDelta) + +for tp in number_types: + enable_cast(tp, StringType) + enable_cast(tp, UnicodeType) + +for tp1 in int_types: + for tp2 in int_types: + if tp1.signed: + if tp2.signed and tp1.basesize() <= tp2.basesize(): + enable_cast(tp1, tp2) + else: + if tp2.signed and tp1.basesize() < tp2.basesize(): + enable_cast(tp1, tp2) + elif not tp2.signed and tp1.basesize() <= tp2.basesize(): + enable_cast(tp1, tp2) +for tp1 in int_types: + for tp2 in float_types + complex_types: + size1 = tp1.basesize() + size2 = tp2.basesize() + if (size1 < 8 and size2 > size1) or (size1 >= 8 and size2 >= size1): + enable_cast(tp1, tp2) +for tp1 in float_types: + for tp2 in float_types + complex_types: + if tp1.basesize() <= tp2.basesize(): + enable_cast(tp1, tp2) +for tp1 in complex_types: + for tp2 in complex_types: + if tp1.basesize() <= tp2.basesize(): + enable_cast(tp1, tp2) From noreply at buildbot.pypy.org Sat Apr 25 05:41:08 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Sat, 25 Apr 2015 05:41:08 +0200 (CEST) Subject: [pypy-commit] pypy can_cast: implement casting=no and casting=equiv Message-ID: <20150425034108.1D28E1C022E@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: can_cast Changeset: r76931:48070170c83b Date: 2015-04-25 04:00 +0100 http://bitbucket.org/pypy/pypy/changeset/48070170c83b/ Log: implement casting=no and casting=equiv diff --git a/pypy/module/micronumpy/arrayops.py b/pypy/module/micronumpy/arrayops.py --- a/pypy/module/micronumpy/arrayops.py +++ b/pypy/module/micronumpy/arrayops.py @@ -304,7 +304,15 @@ def can_cast(space, w_from, w_totype, casting='safe'): target = as_dtype(space, w_totype) origin = as_dtype(space, w_from) # XXX - return space.wrap(origin.can_cast_to(target)) + return space.wrap(can_cast_type(space, origin, target, casting)) + +def can_cast_type(space, origin, target, casting): + if casting == 'no': + return origin.eq(space, target) + elif casting == 'equiv': + return origin.num == target.num and origin.elsize == target.elsize + else: + return origin.can_cast_to(target) def as_dtype(space, w_arg): From noreply at buildbot.pypy.org Sat Apr 25 05:41:09 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Sat, 25 Apr 2015 05:41:09 +0200 (CEST) Subject: [pypy-commit] pypy can_cast: implement casting=unsafe and casting=same_kind Message-ID: <20150425034109.5F0E01C022E@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: can_cast Changeset: r76932:c34ce46667f0 Date: 2015-04-25 04:41 +0100 http://bitbucket.org/pypy/pypy/changeset/c34ce46667f0/ Log: implement casting=unsafe and casting=same_kind diff --git a/pypy/module/micronumpy/arrayops.py b/pypy/module/micronumpy/arrayops.py --- a/pypy/module/micronumpy/arrayops.py +++ b/pypy/module/micronumpy/arrayops.py @@ -5,9 +5,11 @@ constants as NPY from pypy.module.micronumpy.base import convert_to_array, W_NDimArray from pypy.module.micronumpy.converters import clipmode_converter -from pypy.module.micronumpy.strides import Chunk, Chunks, shape_agreement, \ - shape_agreement_multiple +from pypy.module.micronumpy.strides import ( + Chunk, Chunks, shape_agreement, shape_agreement_multiple) from .boxes import W_GenericBox +from .types import ( + Bool, ULong, Long, Float64, Complex64, UnicodeType, VoidType, ObjectType) def where(space, w_arr, w_x=None, w_y=None): @@ -306,11 +308,25 @@ origin = as_dtype(space, w_from) # XXX return space.wrap(can_cast_type(space, origin, target, casting)) +kind_ordering = { + Bool.kind: 0, ULong.kind: 1, Long.kind: 2, + Float64.kind: 4, Complex64.kind: 5, + NPY.STRINGLTR: 6, NPY.STRINGLTR2: 6, + UnicodeType.kind: 7, VoidType.kind: 8, ObjectType.kind: 9} + def can_cast_type(space, origin, target, casting): if casting == 'no': return origin.eq(space, target) elif casting == 'equiv': return origin.num == target.num and origin.elsize == target.elsize + elif casting == 'unsafe': + return True + elif casting == 'same_kind': + if origin.can_cast_to(target): + return True + if origin.kind in kind_ordering and target.kind in kind_ordering: + return kind_ordering[origin.kind] <= kind_ordering[target.kind] + return False else: return origin.can_cast_to(target) From noreply at buildbot.pypy.org Sat Apr 25 10:56:36 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 25 Apr 2015 10:56:36 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: ffi.callback() Message-ID: <20150425085636.346CA1C1207@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1808:6b749854ff50 Date: 2015-04-25 10:57 +0200 http://bitbucket.org/cffi/cffi/changeset/6b749854ff50/ Log: ffi.callback() diff --git a/_cffi1/ffi_obj.c b/_cffi1/ffi_obj.c --- a/_cffi1/ffi_obj.c +++ b/_cffi1/ffi_obj.c @@ -94,6 +94,7 @@ #define ACCEPT_CTYPE 2 #define ACCEPT_CDATA 4 #define ACCEPT_ALL (ACCEPT_STRING | ACCEPT_CTYPE | ACCEPT_CDATA) +#define CONSIDER_FN_AS_FNPTR 8 static CTypeDescrObject *_ffi_type(FFIObject *ffi, PyObject *arg, int accept) @@ -120,8 +121,14 @@ input_text, spaces); return NULL; } - CTypeDescrObject *ct = realize_c_type(ffi->types_builder, - ffi->info.output, index); + CTypeDescrObject *ct; + if (accept & CONSIDER_FN_AS_FNPTR) { + ct = realize_c_type_fn_as_fnptr(ffi->types_builder, + ffi->info.output, index); + } + else { + ct = realize_c_type(ffi->types_builder, ffi->info.output, index); + } if (ct == NULL) return NULL; @@ -505,6 +512,55 @@ } #endif +PyDoc_STRVAR(ffi_callback_doc, +"Return a callback object or a decorator making such a callback object.\n" +"'cdecl' must name a C function pointer type. The callback invokes the\n" +"specified 'python_callable' (which may be provided either directly or\n" +"via a decorator). Important: the callback object must be manually\n" +"kept alive for as long as the callback may be invoked from the C code."); + +static PyObject *_ffi_callback_decorator(PyObject *outer_args, PyObject *fn) +{ + PyObject *res, *old; + + old = PyTuple_GET_ITEM(outer_args, 1); + PyTuple_SET_ITEM(outer_args, 1, fn); + res = b_callback(NULL, outer_args); + PyTuple_SET_ITEM(outer_args, 1, old); + return res; +} + +static PyObject *ffi_callback(FFIObject *self, PyObject *args, PyObject *kwds) +{ + PyObject *cdecl, *python_callable = Py_None, *error = Py_None; + PyObject *res; + static char *keywords[] = {"cdecl", "python_callable", "error", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|OO", keywords, + &cdecl, &python_callable, &error)) + return NULL; + + cdecl = (PyObject *)_ffi_type(self, cdecl, ACCEPT_STRING | ACCEPT_CDATA | + CONSIDER_FN_AS_FNPTR); + if (cdecl == NULL) + return NULL; + + args = Py_BuildValue("(OOO)", cdecl, python_callable, error); + if (args == NULL) + return NULL; + + if (python_callable != Py_None) { + res = b_callback(NULL, args); + } + else { + static PyMethodDef md = {"callback_decorator", + (PyCFunction)_ffi_callback_decorator, METH_O}; + res = PyCFunction_New(&md, args); + } + Py_DECREF(args); + return res; +} + PyDoc_STRVAR(ffi_errno_doc, "the value of 'errno' from/to the C calls"); static PyObject *ffi_get_errno(PyObject *self, void *closure) @@ -610,27 +666,27 @@ } static PyMethodDef ffi_methods[] = { - {"__set_types", (PyCFunction)ffi__set_types,METH_VARARGS}, + {"__set_types",(PyCFunction)ffi__set_types, METH_VARARGS}, #if 0 - {"addressof", (PyCFunction)ffi_addressof, METH_VARARGS}, + {"addressof", (PyCFunction)ffi_addressof, METH_VARARGS}, #endif - {"alignof", (PyCFunction)ffi_alignof, METH_O, ffi_alignof_doc}, - {"cast", (PyCFunction)ffi_cast, METH_VARARGS, ffi_cast_doc}, + {"alignof", (PyCFunction)ffi_alignof, METH_O, ffi_alignof_doc}, + {"callback", (PyCFunction)ffi_callback, METH_VARARGS | + METH_KEYWORDS,ffi_callback_doc}, + {"cast", (PyCFunction)ffi_cast, METH_VARARGS, ffi_cast_doc}, #if 0 - {"close_library", ffi_close_library, METH_VARARGS | METH_STATIC}, - {"from_handle", (PyCFunction)ffi_from_handle,METH_O}, - {"gc", (PyCFunction)ffi_gc, METH_VARARGS}, - {"getctype", (PyCFunction)ffi_getctype, METH_VARARGS}, - {"load_library", (PyCFunction)ffi_load_library,METH_VARARGS|METH_KEYWORDS}, + {"from_handle",(PyCFunction)ffi_from_handle,METH_O}, + {"gc", (PyCFunction)ffi_gc, METH_VARARGS}, + {"getctype", (PyCFunction)ffi_getctype, METH_VARARGS}, #endif - {"offsetof", (PyCFunction)ffi_offsetof, METH_VARARGS,ffi_offsetof_doc}, - {"new", (PyCFunction)ffi_new, METH_VARARGS, ffi_new_doc}, + {"offsetof", (PyCFunction)ffi_offsetof, METH_VARARGS, ffi_offsetof_doc}, + {"new", (PyCFunction)ffi_new, METH_VARARGS, ffi_new_doc}, #if 0 - {"new_handle", (PyCFunction)ffi_new_handle,METH_O}, + {"new_handle", (PyCFunction)ffi_new_handle, METH_O}, #endif - {"sizeof", (PyCFunction)ffi_sizeof, METH_O, ffi_sizeof_doc}, - {"string", (PyCFunction)ffi_string, METH_VARARGS, ffi_string_doc}, - {"typeof", (PyCFunction)ffi_typeof, METH_O, ffi_typeof_doc}, + {"sizeof", (PyCFunction)ffi_sizeof, METH_O, ffi_sizeof_doc}, + {"string", (PyCFunction)ffi_string, METH_VARARGS, ffi_string_doc}, + {"typeof", (PyCFunction)ffi_typeof, METH_O, ffi_typeof_doc}, {NULL} }; diff --git a/_cffi1/realize_c_type.c b/_cffi1/realize_c_type.c --- a/_cffi1/realize_c_type.c +++ b/_cffi1/realize_c_type.c @@ -266,6 +266,27 @@ } } +/* Same as realize_c_type(), but if it's a function type, return the + corresponding function pointer ctype instead of complaining. +*/ +static CTypeDescrObject * +realize_c_type_fn_as_fnptr(builder_c_t *builder, + _cffi_opcode_t opcodes[], int index) +{ + PyObject *x = _realize_c_type_or_func(builder, opcodes, index); + if (x == NULL || CTypeDescr_Check(x)) { + return (CTypeDescrObject *)x; + } + else { + PyObject *y; + assert(PyTuple_Check(x)); + y = PyTuple_GET_ITEM(x, 0); + Py_INCREF(y); + Py_DECREF(x); + return (CTypeDescrObject *)y; + } +} + static PyObject * _realize_c_type_or_func(builder_c_t *builder, _cffi_opcode_t opcodes[], int index) diff --git a/_cffi1/recompiler.py b/_cffi1/recompiler.py --- a/_cffi1/recompiler.py +++ b/_cffi1/recompiler.py @@ -762,4 +762,7 @@ attr = getattr(module.ffi, name) if attr is not getattr(ffi, name, object()): setattr(ffi, name, attr) + def typeof_disabled(*args, **kwds): + raise NotImplementedError + ffi._typeof = typeof_disabled return module.lib diff --git a/_cffi1/test_ffi_obj.py b/_cffi1/test_ffi_obj.py --- a/_cffi1/test_ffi_obj.py +++ b/_cffi1/test_ffi_obj.py @@ -82,3 +82,17 @@ assert ffi.sizeof("int[41]") == 41 * 4 assert ffi.sizeof(ffi.new("int[41]")) == 41 * 4 assert ffi.sizeof(ffi.new("int[]", 41)) == 41 * 4 + +def test_ffi_callback(): + ffi = _cffi1_backend.FFI() + assert ffi.callback("int(int)", lambda x: x + 42)(10) == 52 + assert ffi.callback("int(*)(int)", lambda x: x + 42)(10) == 52 + assert ffi.callback("int(int)", lambda x: x + "", -66)(10) == -66 + assert ffi.callback("int(int)", lambda x: x + "", error=-66)(10) == -66 + +def test_ffi_callback_decorator(): + ffi = _cffi1_backend.FFI() + assert ffi.callback("int(*)(int)")(lambda x: x + 42)(10) == 52 + deco = ffi.callback("int(int)", error=-66) + assert deco(lambda x: x + "")(10) == -66 + assert deco(lambda x: x + 42)(10) == 52 From noreply at buildbot.pypy.org Sat Apr 25 11:04:18 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 25 Apr 2015 11:04:18 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: remove a test that doesn't make sense Message-ID: <20150425090418.BA5BF1C1207@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1809:6055d7106150 Date: 2015-04-25 11:04 +0200 http://bitbucket.org/cffi/cffi/changeset/6055d7106150/ Log: remove a test that doesn't make sense diff --git a/_cffi1/test_verify1.py b/_cffi1/test_verify1.py --- a/_cffi1/test_verify1.py +++ b/_cffi1/test_verify1.py @@ -898,15 +898,6 @@ lib.cb = my_callback assert lib.foo(4) == 887 -def test_ctypes_backend_forces_generic_engine(): - from cffi.backend_ctypes import CTypesBackend - ffi = FFI(backend=CTypesBackend()) - ffi.cdef("int func(int a);") - lib = ffi.verify("int func(int a) { return a * 42; }") - assert not hasattr(lib, '_cffi_python_module') - assert hasattr(lib, '_cffi_generic_module') - assert lib.func(100) == 4200 - def test_call_with_struct_ptr(): ffi = FFI() ffi.cdef("typedef struct { int x; ...; } foo_t; int foo(foo_t *);") From noreply at buildbot.pypy.org Sat Apr 25 11:04:19 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 25 Apr 2015 11:04:19 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: ellipsis Message-ID: <20150425090419.C09991C1207@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1810:c818c7444718 Date: 2015-04-25 11:04 +0200 http://bitbucket.org/cffi/cffi/changeset/c818c7444718/ Log: ellipsis diff --git a/_cffi1/realize_c_type.c b/_cffi1/realize_c_type.c --- a/_cffi1/realize_c_type.c +++ b/_cffi1/realize_c_type.c @@ -489,7 +489,7 @@ case _CFFI_OP_FUNCTION: { PyObject *fargs; - int i, base_index, num_args; + int i, base_index, num_args, ellipsis; y = (PyObject *)realize_c_type(builder, opcodes, _CFFI_GETARG(op)); if (y == NULL) @@ -501,6 +501,8 @@ _CFFI_OP_FUNCTION_END) num_args++; + ellipsis = _CFFI_GETARG(opcodes[base_index + num_args]) & 1; + fargs = PyTuple_New(num_args); if (fargs == NULL) { Py_DECREF(y); @@ -517,7 +519,8 @@ PyTuple_SET_ITEM(fargs, i, z); } - z = new_function_type(fargs, (CTypeDescrObject *)y, 0, FFI_DEFAULT_ABI); + z = new_function_type(fargs, (CTypeDescrObject *)y, ellipsis, + FFI_DEFAULT_ABI); z = get_unique_type(builder, z); Py_DECREF(fargs); Py_DECREF(y); diff --git a/_cffi1/recompiler.py b/_cffi1/recompiler.py --- a/_cffi1/recompiler.py +++ b/_cffi1/recompiler.py @@ -312,6 +312,8 @@ def _generate_cpy_function_collecttype(self, tp, name): self._do_collect_type(tp.as_raw_function()) + if tp.ellipsis: + self._do_collect_type(tp) def _generate_cpy_function_decl(self, tp, name): assert isinstance(tp, model.FunctionPtrType) @@ -319,7 +321,7 @@ # cannot support vararg functions better than this: check for its # exact type (including the fixed arguments), and build it as a # constant function pointer (no CPython wrapper) - self._generate_cpy_const(False, name, tp) + self._generate_cpy_constant_decl(tp, name) return prnt = self._prnt numargs = len(tp.args) @@ -388,7 +390,8 @@ def _generate_cpy_function_ctx(self, tp, name): if tp.ellipsis: - XXX + self._generate_cpy_constant_ctx(tp, name) + return type_index = self._typesdict[tp.as_raw_function()] numargs = len(tp.args) if numargs == 0: From noreply at buildbot.pypy.org Sat Apr 25 11:15:48 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 25 Apr 2015 11:15:48 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: fix test Message-ID: <20150425091548.855401C00F7@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1811:d3fe2c3910db Date: 2015-04-25 11:07 +0200 http://bitbucket.org/cffi/cffi/changeset/d3fe2c3910db/ Log: fix test diff --git a/_cffi1/test_verify1.py b/_cffi1/test_verify1.py --- a/_cffi1/test_verify1.py +++ b/_cffi1/test_verify1.py @@ -1027,7 +1027,7 @@ ffi = FFI() ffi.cdef("struct foo_s { long a; ...; };\n" "int (*foo)(struct foo_s);") - e = py.test.raises(TypeError, ffi.verify, """ + lib = ffi.verify(""" struct foo_s { double b; long a; @@ -1037,6 +1037,7 @@ } int (*foo)(struct foo_s s) = &foo1; """) + e = py.test.raises(TypeError, "lib.foo") # lazily msg ='cannot pass as an argument a struct that was completed with verify()' assert msg in str(e.value) From noreply at buildbot.pypy.org Sat Apr 25 11:15:49 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 25 Apr 2015 11:15:49 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: fix Message-ID: <20150425091549.7D4B71C00F7@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1812:7a25149295e0 Date: 2015-04-25 11:16 +0200 http://bitbucket.org/cffi/cffi/changeset/7a25149295e0/ Log: fix diff --git a/_cffi1/test_recompiler.py b/_cffi1/test_recompiler.py --- a/_cffi1/test_recompiler.py +++ b/_cffi1/test_recompiler.py @@ -51,7 +51,7 @@ def test_type_table_variadic_function(): check_type_table("int sin(int, ...);", - "(FUNCTION 1)(PRIMITIVE 7)(FUNCTION_END 1)") + "(FUNCTION 1)(PRIMITIVE 7)(FUNCTION_END 1)(POINTER 0)") def test_type_table_array(): check_type_table("int a[100];", From noreply at buildbot.pypy.org Sat Apr 25 11:15:50 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 25 Apr 2015 11:15:50 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: A clearer way to complain Message-ID: <20150425091550.7CBB71C00F7@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1813:bdb89e5e392e Date: 2015-04-25 11:16 +0200 http://bitbucket.org/cffi/cffi/changeset/bdb89e5e392e/ Log: A clearer way to complain diff --git a/_cffi1/recompiler.py b/_cffi1/recompiler.py --- a/_cffi1/recompiler.py +++ b/_cffi1/recompiler.py @@ -142,6 +142,7 @@ # # call generate_cpy_xxx_decl(), for every xxx found from # ffi._parser._declarations. This generates all the functions. + self._seen_constants = set() self._generate("decl") # # the declaration of '_cffi_globals' and '_cffi_typenames' @@ -567,6 +568,11 @@ # constants, declared with "static const ..." def _generate_cpy_const(self, is_int, name, tp=None, category='const'): + if (category, name) in self._seen_constants: + raise ffiplatform.VerificationError( + "duplicate declaration of %s '%s'" % (category, name)) + self._seen_constants.add((category, name)) + # prnt = self._prnt funcname = '_cffi_%s_%s' % (category, name) if is_int: diff --git a/_cffi1/test_recompiler.py b/_cffi1/test_recompiler.py --- a/_cffi1/test_recompiler.py +++ b/_cffi1/test_recompiler.py @@ -1,5 +1,5 @@ import sys, py -from cffi import FFI +from cffi import FFI, VerificationError from _cffi1 import recompiler @@ -277,7 +277,13 @@ assert lib.B2 == 1 assert ffi.sizeof("enum e1") == ffi.sizeof("long") assert ffi.sizeof("enum e2") == ffi.sizeof("int") - + +def test_duplicate_enum(): + ffi = FFI() + ffi.cdef("enum e1 { A1, ... }; enum e2 { A1, ... };") + py.test.raises(VerificationError, verify, ffi, 'test_duplicate_enum', + "enum e1 { A1 }; enum e2 { B1 };") + def test_dotdotdot_length_of_array_field(): ffi = FFI() ffi.cdef("struct foo_s { int a[...]; int b[...]; };") From noreply at buildbot.pypy.org Sat Apr 25 11:42:00 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 25 Apr 2015 11:42:00 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: fix: partially drop checking for internal consistency, which was bogus Message-ID: <20150425094200.9F7311C022E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1814:fd28da258879 Date: 2015-04-25 11:38 +0200 http://bitbucket.org/cffi/cffi/changeset/fd28da258879/ Log: fix: partially drop checking for internal consistency, which was bogus diff --git a/_cffi1/realize_c_type.c b/_cffi1/realize_c_type.c --- a/_cffi1/realize_c_type.c +++ b/_cffi1/realize_c_type.c @@ -448,8 +448,9 @@ PyTuple_SET_ITEM(enumerators, i, tmp); gindex = search_in_globals(&builder->ctx, p, j); + assert(gindex >= 0); g = &builder->ctx.globals[gindex]; - assert(gindex >= 0 && g->type_op == op); + assert(g->type_op == _CFFI_OP(_CFFI_OP_ENUM, -1)); tmp = realize_global_int(g); PyTuple_SET_ITEM(enumvalues, i, tmp); diff --git a/_cffi1/recompiler.py b/_cffi1/recompiler.py --- a/_cffi1/recompiler.py +++ b/_cffi1/recompiler.py @@ -619,7 +619,7 @@ def _enum_ctx(self, tp, cname): type_index = self._typesdict[tp] - type_op = '_CFFI_OP(_CFFI_OP_ENUM, %d)' % type_index + type_op = '_CFFI_OP(_CFFI_OP_ENUM, -1)' for enumerator in tp.enumerators: self._lsts["global"].append( ' { "%s", _cffi_const_%s, %s },' % (enumerator, enumerator, From noreply at buildbot.pypy.org Sat Apr 25 11:42:01 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 25 Apr 2015 11:42:01 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: more test fixes Message-ID: <20150425094201.BF37F1C022E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1815:c2023e4a4a6d Date: 2015-04-25 11:42 +0200 http://bitbucket.org/cffi/cffi/changeset/c2023e4a4a6d/ Log: more test fixes diff --git a/_cffi1/test_verify1.py b/_cffi1/test_verify1.py --- a/_cffi1/test_verify1.py +++ b/_cffi1/test_verify1.py @@ -1139,10 +1139,12 @@ assert lib.BB == 1 def test_typedef_broken_complete_enum(): + # xxx this is broken in old cffis, but works with recompiler.py ffi = FFI() ffi.cdef("typedef enum { AA, BB } enum1_t;") - py.test.raises(VerificationError, ffi.verify, - "typedef enum { AA, CC, BB } enum1_t;") + lib = ffi.verify("typedef enum { AA, CC, BB } enum1_t;") + assert lib.AA == 0 + assert lib.BB == 2 def test_typedef_incomplete_enum(): ffi = FFI() @@ -1723,18 +1725,17 @@ def test_enum_bug118(): maxulong = 256 ** FFI().sizeof("unsigned long") - 1 - for c1, c2, c2c in [(0xffffffff, -1, ''), - (maxulong, -1, ''), - (-1, 0xffffffff, 'U'), - (-1, maxulong, 'UL')]: + for c2, c2c in [(-1, ''), + (-1, ''), + (0xffffffff, 'U'), + (maxulong, 'UL'), + (-maxulong / 3, 'L')]: if c2c and sys.platform == 'win32': continue # enums may always be signed with MSVC ffi = FFI() - ffi.cdef("enum foo_e { AA=%s };" % c1) - e = py.test.raises(VerificationError, ffi.verify, - "enum foo_e { AA=%s%s };" % (c2, c2c)) - assert str(e.value) == ('enum foo_e: AA has the real value %d, not %d' - % (c2, c1)) + ffi.cdef("enum foo_e { AA };") + lib = ffi.verify("enum foo_e { AA=%s%s };" % (c2, c2c)) + assert lib.AA == c2 def test_string_to_voidp_arg(): ffi = FFI() From noreply at buildbot.pypy.org Sat Apr 25 12:05:23 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 25 Apr 2015 12:05:23 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: fixes for anonymous nested structs Message-ID: <20150425100523.B98621C1207@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1816:d92f0d7cfb50 Date: 2015-04-25 12:06 +0200 http://bitbucket.org/cffi/cffi/changeset/d92f0d7cfb50/ Log: fixes for anonymous nested structs diff --git a/_cffi1/recompiler.py b/_cffi1/recompiler.py --- a/_cffi1/recompiler.py +++ b/_cffi1/recompiler.py @@ -78,7 +78,7 @@ self._do_collect_type(tp.as_raw_function()) elif isinstance(tp, model.StructOrUnion): if tp.fldtypes is not None: - for name1, tp1 in zip(tp.fldnames, tp.fldtypes): + for name1, tp1, _ in tp.enumfields(): self._do_collect_type(self._field_type(tp, name1, tp1)) else: for _, x in tp._get_items(): @@ -448,14 +448,15 @@ def _struct_ctx(self, tp, cname, approxname): type_index = self._typesdict[tp] flags = [] - if tp.partial: + if tp.partial or tp.has_anonymous_struct_fields(): flags.append('CT_CUSTOM_FIELD_POS') if isinstance(tp, model.UnionType): flags.append('CT_UNION') flags = ('|'.join(flags)) or '0' if tp.fldtypes is not None: c_field = [approxname] - for fldname, fldtype, fbitsize in tp.enumfields(): + enumfields = list(tp.enumfields()) + for fldname, fldtype, fbitsize in enumfields: fldtype = self._field_type(tp, fldname, fldtype) spaces = " " * len(fldname) # cname is None for _add_missing_struct_unions() only @@ -483,13 +484,13 @@ if cname is None: # unknown name, for _add_missing_struct_unions size_align = (' (size_t)-2, -2, /* unnamed */\n' + ' _cffi_FIELDS_FOR_%s, %d },' % (approxname, - len(tp.fldtypes),)) + len(enumfields),)) else: size_align = ('\n' + ' sizeof(%s),\n' % (cname,) + ' offsetof(struct _cffi_align_%s, y),\n'% (approxname,) + ' _cffi_FIELDS_FOR_%s, %d },' % (approxname, - len(tp.fldtypes),)) + len(enumfields),)) else: size_align = ' (size_t)-1, -1, -1, 0 /* opaque */ },' self._lsts["struct_union"].append( diff --git a/_cffi1/test_verify1.py b/_cffi1/test_verify1.py --- a/_cffi1/test_verify1.py +++ b/_cffi1/test_verify1.py @@ -1282,6 +1282,8 @@ ffi.cdef(""" struct foo_s { struct { int a; char b; }; union { char c, d; }; }; """) + assert ffi.offsetof("struct foo_s", "c") == 2 * ffi.sizeof("int") + assert ffi.sizeof("struct foo_s") == 3 * ffi.sizeof("int") ffi.verify(""" struct foo_s { struct { int a; char b; }; union { char c, d; }; }; """) @@ -1305,9 +1307,10 @@ py.test.raises(VerificationError, ffi.verify, """ struct foo_s { struct { int a; short b; }; union { char c, d; }; }; """) - py.test.raises(VerificationError, ffi.verify, """ - struct foo_s { struct { int a; char e, b; }; union { char c, d; }; }; - """) + # works fine now + #py.test.raises(VerificationError, ffi.verify, """ + # struct foo_s { struct { int a; char e, b; }; union { char c, d; }; }; + #""") def test_nested_anonymous_struct_inexact_1(): ffi = FFI() diff --git a/cffi/model.py b/cffi/model.py --- a/cffi/model.py +++ b/cffi/model.py @@ -271,6 +271,12 @@ self.fldbitsize = fldbitsize self.build_c_name_with_marker() + def has_anonymous_struct_fields(self): + for name, type in zip(self.fldnames, self.fldtypes): + if name == '' and isinstance(type, StructOrUnion): + return True + return False + def enumfields(self): for name, type, bitsize in zip(self.fldnames, self.fldtypes, self.fldbitsize): From noreply at buildbot.pypy.org Sat Apr 25 12:06:08 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 25 Apr 2015 12:06:08 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: oups Message-ID: <20150425100608.E845F1C1207@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1817:0da180ccac93 Date: 2015-04-25 12:06 +0200 http://bitbucket.org/cffi/cffi/changeset/0da180ccac93/ Log: oups diff --git a/cffi/model.py b/cffi/model.py --- a/cffi/model.py +++ b/cffi/model.py @@ -272,6 +272,8 @@ self.build_c_name_with_marker() def has_anonymous_struct_fields(self): + if self.fldtypes is None: + return False for name, type in zip(self.fldnames, self.fldtypes): if name == '' and isinstance(type, StructOrUnion): return True From noreply at buildbot.pypy.org Sat Apr 25 12:06:38 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 25 Apr 2015 12:06:38 +0200 (CEST) Subject: [pypy-commit] cffi default: Add a passing test Message-ID: <20150425100638.880341C1207@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1818:b5bfa871d282 Date: 2015-04-25 12:07 +0200 http://bitbucket.org/cffi/cffi/changeset/b5bfa871d282/ Log: Add a passing test diff --git a/testing/backend_tests.py b/testing/backend_tests.py --- a/testing/backend_tests.py +++ b/testing/backend_tests.py @@ -1388,6 +1388,17 @@ assert p.c == 14 assert p.d == 14 + def test_nested_field_offset_align(self): + ffi = FFI(backend=self.Backend()) + ffi.cdef(""" + struct foo_s { + struct { int a; char b; }; + union { char c; }; + }; + """) + assert ffi.offsetof("struct foo_s", "c") == 2 * SIZE_OF_INT + assert ffi.sizeof("struct foo_s") == 3 * SIZE_OF_INT + def test_nested_anonymous_union(self): ffi = FFI(backend=self.Backend()) ffi.cdef(""" diff --git a/testing/test_ctypes.py b/testing/test_ctypes.py --- a/testing/test_ctypes.py +++ b/testing/test_ctypes.py @@ -28,6 +28,9 @@ def test_nested_anonymous_struct(self): py.test.skip("ctypes backend: not supported: nested anonymous struct") + def test_nested_field_offset_align(self): + py.test.skip("ctypes backend: not supported: nested anonymous struct") + def test_nested_anonymous_union(self): py.test.skip("ctypes backend: not supported: nested anonymous union") From noreply at buildbot.pypy.org Sat Apr 25 12:12:24 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 25 Apr 2015 12:12:24 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: fix/skip Message-ID: <20150425101225.0134D1C00F7@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1819:4744cc882ca2 Date: 2015-04-25 12:13 +0200 http://bitbucket.org/cffi/cffi/changeset/4744cc882ca2/ Log: fix/skip diff --git a/_cffi1/recompiler.py b/_cffi1/recompiler.py --- a/_cffi1/recompiler.py +++ b/_cffi1/recompiler.py @@ -760,9 +760,8 @@ import imp assert module_name not in sys.modules, "module name conflict: %r" % ( module_name,) - outputfilename = recompile(ffi, module_name, preamble, - tmpdir=str(udir), - *args, **kwds) + kwds.setdefault('tmpdir', str(udir)) + outputfilename = recompile(ffi, module_name, preamble, *args, **kwds) module = imp.load_dynamic(module_name, outputfilename) # # hack hack hack: copy all *bound methods* from module.ffi back to the diff --git a/_cffi1/test_verify1.py b/_cffi1/test_verify1.py --- a/_cffi1/test_verify1.py +++ b/_cffi1/test_verify1.py @@ -27,12 +27,11 @@ _verify_counter = 0 def verify(self, preamble='', *args, **kwds): - from _cffi1.udir import udir FFI._verify_counter += 1 return recompiler.verify(self, 'verify%d' % FFI._verify_counter, preamble, *args, extra_compile_args=self._extra_compile_args, - tmp=str(udir), **kwds) + **kwds) class FFI_warnings_not_error(FFI): _extra_compile_args = [] @@ -1390,6 +1389,7 @@ assert lib.foo(100) == 142 def test_relative_to(): + py.test.skip("not available") import tempfile, os from testing.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) From noreply at buildbot.pypy.org Sat Apr 25 12:24:57 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 25 Apr 2015 12:24:57 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Remove the dependency on the header files in _cffi1 Message-ID: <20150425102457.354F61C12E0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1820:a721107bac60 Date: 2015-04-25 12:25 +0200 http://bitbucket.org/cffi/cffi/changeset/a721107bac60/ Log: Remove the dependency on the header files in _cffi1 diff --git a/_cffi1/parse_c_type.c b/_cffi1/parse_c_type.c --- a/_cffi1/parse_c_type.c +++ b/_cffi1/parse_c_type.c @@ -2,6 +2,8 @@ #include #include #include + +#define _CFFI_INTERNAL #include "parse_c_type.h" diff --git a/_cffi1/parse_c_type.h b/_cffi1/parse_c_type.h --- a/_cffi1/parse_c_type.h +++ b/_cffi1/parse_c_type.h @@ -69,17 +69,18 @@ struct _cffi_struct_union_s { const char *name; int type_index; // -> _cffi_types, on a OP_STRUCT_UNION - int flags; // CT_UNION? CT_IS_OPAQUE? + int flags; // CT_UNION? CT_CUSTOM_FIELD_POS? size_t size; int alignment; int first_field_index; // -> _cffi_fields array int num_fields; }; +#ifdef _CFFI_INTERNAL #define CT_UNION 128 -#define CT_IS_OPAQUE 4096 #define CT_CUSTOM_FIELD_POS 32768 /* ^^^ if not CUSTOM_FIELD_POS, complain if fields are not in the "standard layout" and/or if some are missing */ +#endif struct _cffi_field_s { const char *name; @@ -122,8 +123,10 @@ const char *error_message; }; +#ifdef _CFFI_INTERNAL int parse_c_type(struct _cffi_parse_info_s *info, const char *input); int search_in_globals(const struct _cffi_type_context_s *ctx, const char *search, size_t search_len); int search_in_struct_unions(const struct _cffi_type_context_s *ctx, const char *search, size_t search_len); +#endif diff --git a/_cffi1/recompiler.py b/_cffi1/recompiler.py --- a/_cffi1/recompiler.py +++ b/_cffi1/recompiler.py @@ -111,12 +111,21 @@ # a KeyError here is a bug. please report it! :-) return self._typesdict[type] + def _rel_readlines(self, filename): + g = open(os.path.join(os.path.dirname(__file__), filename), 'r') + lines = g.readlines() + g.close() + return lines + def write_source_to_f(self, f, preamble): self._f = f prnt = self._prnt # - # first the '#include' - prnt('#include "_cffi_include.h"') + # first the '#include' (actually done by inlining the file's content) + lines = self._rel_readlines('_cffi_include.h') + i = lines.index('#include "parse_c_type.h"\n') + lines[i:i+1] = self._rel_readlines('parse_c_type.h') + prnt(''.join(lines)) # # then paste the C source given by the user, verbatim. prnt('/************************************************************/') @@ -447,12 +456,11 @@ def _struct_ctx(self, tp, cname, approxname): type_index = self._typesdict[tp] - flags = [] + flags = 0 if tp.partial or tp.has_anonymous_struct_fields(): - flags.append('CT_CUSTOM_FIELD_POS') + flags |= 32768 # CT_CUSTOM_FIELD_POS if isinstance(tp, model.UnionType): - flags.append('CT_UNION') - flags = ('|'.join(flags)) or '0' + flags |= 128 # CT_UNION if tp.fldtypes is not None: c_field = [approxname] enumfields = list(tp.enumfields()) @@ -494,7 +502,7 @@ else: size_align = ' (size_t)-1, -1, -1, 0 /* opaque */ },' self._lsts["struct_union"].append( - ' { "%s", %d, %s,' % (tp.name, type_index, flags) + size_align) + ' { "%s", %d, 0x%x,' % (tp.name, type_index, flags) + size_align) self._seen_struct_unions.add(tp) def _add_missing_struct_unions(self): @@ -744,8 +752,6 @@ def _get_extension(module_name, c_file, kwds): source_name = ffiplatform.maybe_relative_path(c_file) - include_dirs = kwds.setdefault('include_dirs', []) - include_dirs.insert(0, '.') # XXX return ffiplatform.get_extension(source_name, module_name, **kwds) def recompile(ffi, module_name, preamble, tmpdir='.', **kwds): diff --git a/_cffi1/test_parse_c_type.py b/_cffi1/test_parse_c_type.py --- a/_cffi1/test_parse_c_type.py +++ b/_cffi1/test_parse_c_type.py @@ -5,9 +5,11 @@ r_macro = re.compile(r"#define \w+[(][^\n]*|#include [^\n]*") r_define = re.compile(r"(#define \w+) [^\n]*") +r_ifdefs = re.compile(r"(#ifdef |#endif)[^\n]*") header = open('parse_c_type.h').read() header = r_macro.sub(r"", header) header = r_define.sub(r"\1 ...", header) +header = r_ifdefs.sub(r"", header) ffi = cffi.FFI() ffi.cdef(header) From noreply at buildbot.pypy.org Sat Apr 25 12:35:44 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 25 Apr 2015 12:35:44 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: fix Message-ID: <20150425103544.E19EC1C12E0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1821:17a39c154ed7 Date: 2015-04-25 12:36 +0200 http://bitbucket.org/cffi/cffi/changeset/17a39c154ed7/ Log: fix diff --git a/_cffi1/recompiler.py b/_cffi1/recompiler.py --- a/_cffi1/recompiler.py +++ b/_cffi1/recompiler.py @@ -532,23 +532,19 @@ self._struct_collecttype(tp) _generate_cpy_union_collecttype = _generate_cpy_struct_collecttype + def _struct_names(self, tp): + cname = tp.get_c_name('') + if ' ' in cname: + return cname, cname.replace(' ', '_') + else: + return cname, '_' + cname + def _generate_cpy_struct_decl(self, tp, name): - cname = tp._get_c_name() - if ' ' in cname: - prefix, declname = cname.split(' ', 1) - else: - prefix, declname = '', cname - while declname.startswith('$'): - prefix += 'D' - declname = declname[1:] - approxname = prefix + '_' + declname - assert '$' not in approxname - self._struct_decl(tp, cname, approxname) + self._struct_decl(tp, *self._struct_names(tp)) _generate_cpy_union_decl = _generate_cpy_struct_decl - def _generate_cpy_struct_ctx(self, tp, name, prefix='s'): - cname = tp.get_c_name('') - self._struct_ctx(tp, cname, cname.replace(' ', '_')) + def _generate_cpy_struct_ctx(self, tp, name): + self._struct_ctx(tp, *self._struct_names(tp)) _generate_cpy_union_ctx = _generate_cpy_struct_ctx # ---------- From noreply at buildbot.pypy.org Sat Apr 25 12:47:31 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 25 Apr 2015 12:47:31 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: ffi.getctype() Message-ID: <20150425104731.400371C00F7@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1822:30d86d0edda2 Date: 2015-04-25 12:48 +0200 http://bitbucket.org/cffi/cffi/changeset/30d86d0edda2/ Log: ffi.getctype() diff --git a/_cffi1/ffi_obj.c b/_cffi1/ffi_obj.c --- a/_cffi1/ffi_obj.c +++ b/_cffi1/ffi_obj.c @@ -414,8 +414,36 @@ return NULL; } } +#endif -static PyObject *ffi_getctype(ZefFFIObject *self, PyObject *args) +static PyObject *_combine_type_name_l(CTypeDescrObject *ct, + size_t extra_text_len) +{ + size_t base_name_len; + PyObject *result; + char *p; + + base_name_len = strlen(ct->ct_name); + result = PyString_FromStringAndSize(NULL, base_name_len + extra_text_len); + if (result == NULL) + return NULL; + + p = PyString_AS_STRING(result); + memcpy(p, ct->ct_name, ct->ct_name_position); + p += ct->ct_name_position; + p += extra_text_len; + memcpy(p, ct->ct_name + ct->ct_name_position, + base_name_len - ct->ct_name_position); + return result; +} + +PyDoc_STRVAR(ffi_getctype_doc, +"Return a string giving the C type 'cdecl', which may be itself a\n" +"string or a object. If 'replace_with' is given, it gives\n" +"extra text to append (or insert for more complicated C types), like a\n" +"variable name, or '*' to get actually the C type 'pointer-to-cdecl'."); + +static PyObject *ffi_getctype(FFIObject *self, PyObject *args) { PyObject *cdecl, *res; char *p, *replace_with = ""; @@ -437,11 +465,11 @@ replace_with_len--; add_paren = (replace_with[0] == '*' && - ((ct->ct_flags & (CT_ARRAY | CT_FUNCTION)) != 0)); + ((ct->ct_flags & CT_ARRAY) != 0)); add_space = (!add_paren && replace_with_len > 0 && replace_with[0] != '[' && replace_with[0] != '('); - res = combine_type_name_l(ct, replace_with_len + add_space + 2 * add_paren); + res = _combine_type_name_l(ct, replace_with_len + add_space + 2*add_paren); if (res == NULL) return NULL; @@ -456,6 +484,7 @@ return res; } +#if 0 static PyObject *ffi_new_handle(ZefFFIObject *self, PyObject *arg) { CTypeDescrObject *ct = ZefNULL->c_type; // @@ -677,8 +706,8 @@ #if 0 {"from_handle",(PyCFunction)ffi_from_handle,METH_O}, {"gc", (PyCFunction)ffi_gc, METH_VARARGS}, - {"getctype", (PyCFunction)ffi_getctype, METH_VARARGS}, #endif + {"getctype", (PyCFunction)ffi_getctype, METH_VARARGS, ffi_getctype_doc}, {"offsetof", (PyCFunction)ffi_offsetof, METH_VARARGS, ffi_offsetof_doc}, {"new", (PyCFunction)ffi_new, METH_VARARGS, ffi_new_doc}, #if 0 diff --git a/_cffi1/test_ffi_obj.py b/_cffi1/test_ffi_obj.py --- a/_cffi1/test_ffi_obj.py +++ b/_cffi1/test_ffi_obj.py @@ -96,3 +96,8 @@ deco = ffi.callback("int(int)", error=-66) assert deco(lambda x: x + "")(10) == -66 assert deco(lambda x: x + 42)(10) == 52 + +def test_ffi_getctype(): + ffi = _cffi1_backend.FFI() + assert ffi.getctype("int*") == "int *" + assert ffi.getctype("int[5]", "a") == "int a[5]" From noreply at buildbot.pypy.org Sat Apr 25 12:48:34 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 25 Apr 2015 12:48:34 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: more tests Message-ID: <20150425104834.797421C00F7@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1823:c9d8791139c0 Date: 2015-04-25 12:49 +0200 http://bitbucket.org/cffi/cffi/changeset/c9d8791139c0/ Log: more tests diff --git a/_cffi1/test_ffi_obj.py b/_cffi1/test_ffi_obj.py --- a/_cffi1/test_ffi_obj.py +++ b/_cffi1/test_ffi_obj.py @@ -99,5 +99,18 @@ def test_ffi_getctype(): ffi = _cffi1_backend.FFI() + assert ffi.getctype("int") == "int" + assert ffi.getctype("int", 'x') == "int x" assert ffi.getctype("int*") == "int *" - assert ffi.getctype("int[5]", "a") == "int a[5]" + assert ffi.getctype("int*", '') == "int *" + assert ffi.getctype("int*", 'x') == "int * x" + assert ffi.getctype("int", '*') == "int *" + assert ffi.getctype("int", ' * x ') == "int * x" + assert ffi.getctype(ffi.typeof("int*"), '*') == "int * *" + assert ffi.getctype("int", '[5]') == "int[5]" + assert ffi.getctype("int[5]", '[6]') == "int[6][5]" + assert ffi.getctype("int[5]", '(*)') == "int(*)[5]" + # special-case for convenience: automatically put '()' around '*' + assert ffi.getctype("int[5]", '*') == "int(*)[5]" + assert ffi.getctype("int[5]", '*foo') == "int(*foo)[5]" + assert ffi.getctype("int[5]", ' ** foo ') == "int(** foo)[5]" From noreply at buildbot.pypy.org Sat Apr 25 14:42:53 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 25 Apr 2015 14:42:53 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: test and fix Message-ID: <20150425124253.428D01C00F7@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1824:38f60cc08b65 Date: 2015-04-25 13:02 +0200 http://bitbucket.org/cffi/cffi/changeset/38f60cc08b65/ Log: test and fix diff --git a/_cffi1/ffi_obj.c b/_cffi1/ffi_obj.c --- a/_cffi1/ffi_obj.c +++ b/_cffi1/ffi_obj.c @@ -569,7 +569,7 @@ &cdecl, &python_callable, &error)) return NULL; - cdecl = (PyObject *)_ffi_type(self, cdecl, ACCEPT_STRING | ACCEPT_CDATA | + cdecl = (PyObject *)_ffi_type(self, cdecl, ACCEPT_STRING | ACCEPT_CTYPE | CONSIDER_FN_AS_FNPTR); if (cdecl == NULL) return NULL; diff --git a/_cffi1/test_ffi_obj.py b/_cffi1/test_ffi_obj.py --- a/_cffi1/test_ffi_obj.py +++ b/_cffi1/test_ffi_obj.py @@ -92,7 +92,7 @@ def test_ffi_callback_decorator(): ffi = _cffi1_backend.FFI() - assert ffi.callback("int(*)(int)")(lambda x: x + 42)(10) == 52 + assert ffi.callback(ffi.typeof("int(*)(int)"))(lambda x: x + 42)(10) == 52 deco = ffi.callback("int(int)", error=-66) assert deco(lambda x: x + "")(10) == -66 assert deco(lambda x: x + 42)(10) == 52 From noreply at buildbot.pypy.org Sat Apr 25 14:42:54 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 25 Apr 2015 14:42:54 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: ffi.addressof() Message-ID: <20150425124254.56DFB1C00F7@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1825:d073836b9835 Date: 2015-04-25 14:43 +0200 http://bitbucket.org/cffi/cffi/changeset/d073836b9835/ Log: ffi.addressof() diff --git a/_cffi1/ffi_obj.c b/_cffi1/ffi_obj.c --- a/_cffi1/ffi_obj.c +++ b/_cffi1/ffi_obj.c @@ -336,85 +336,65 @@ return PyInt_FromSsize_t(offset); } -#if 0 -static PyObject *ffi_addressof(ZefFFIObject *self, PyObject *args) +PyDoc_STRVAR(ffi_addressof_doc, +"With a single arg, return the address of a .\n" +"If 'fields_or_indexes' are given, returns the address of that field or\n" +"array item in the structure or array, recursively in case of nested\n" +"structures."); + +static PyObject *ffi_addressof(FFIObject *self, PyObject *args) { - PyObject *obj; - char *fieldname = NULL; + PyObject *arg, *z, *result; + CTypeDescrObject *ct; + Py_ssize_t i, offset = 0; + int accepted_flags; - if (!PyArg_ParseTuple(args, "O|z:addressof", &obj, &fieldname)) + if (PyTuple_Size(args) < 1) { + PyErr_SetString(PyExc_TypeError, + "addressof() expects at least 1 argument"); + return NULL; + } + + arg = PyTuple_GET_ITEM(args, 0); + ct = _ffi_type(self, arg, ACCEPT_CDATA); + if (ct == NULL) return NULL; - if (CData_Check(obj)) { - CDataObject *cd = (CDataObject *)obj; - CTypeDescrObject *ct; - Py_ssize_t offset; - - ct = cd->c_type; - if (fieldname != NULL && ct->ct_flags & CT_POINTER) - ct = ct->ct_itemdescr; - - if (!(ct->ct_flags & (CT_STRUCT|CT_UNION))) { - PyErr_Format(PyExc_TypeError, - "expected a struct or union cdata, got '%s'", - ct->ct_name); + if (PyTuple_GET_SIZE(args) == 1) { + accepted_flags = CT_STRUCT | CT_UNION | CT_ARRAY; + if ((ct->ct_flags & accepted_flags) == 0) { + PyErr_SetString(PyExc_TypeError, + "expected a cdata struct/union/array object"); return NULL; } - - if (fieldname == NULL) { - offset = 0; - } - else { - CFieldObject *cf = _ffi_field(ct, fieldname); - if (cf == NULL) - return NULL; - offset = cf->cf_offset; - ct = cf->cf_type; - } - ct = fetch_pointer_type(self->types_dict, ct); - if (ct == NULL) - return NULL; - return new_simple_cdata(cd->c_data + offset, ct); } - else if (ZefLib_Check(obj)) { - PyObject *attr, *name; - char *reason; - - if (fieldname == NULL) { - PyErr_SetString(PyExc_TypeError, "addressof(Lib, fieldname) " - "cannot be used with only one argument"); + else { + accepted_flags = CT_STRUCT | CT_UNION | CT_ARRAY | CT_POINTER; + if ((ct->ct_flags & accepted_flags) == 0) { + PyErr_SetString(PyExc_TypeError, + "expected a cdata struct/union/array/pointer object"); return NULL; } - name = PyString_FromString(fieldname); - if (name == NULL) - return NULL; - attr = lib_findattr((ZefLibObject *)obj, name, ZefError); - Py_DECREF(name); - if (attr == NULL) - return NULL; + for (i = 1; i < PyTuple_GET_SIZE(args); i++) { + Py_ssize_t ofs1; + ct = direct_typeoffsetof(ct, PyTuple_GET_ITEM(args, i), + i > 1, &ofs1); + if (ct == NULL) + return NULL; + offset += ofs1; + } + } - if (ZefGlobSupport_Check(attr)) { - return addressof_global_var((ZefGlobSupportObject *)attr); - } + z = new_pointer_type(ct); + z = get_unique_type(self->types_builder, z); + if (z == NULL) + return NULL; - if (PyCFunction_Check(attr)) - reason = "declare that function as a function pointer instead"; - else - reason = "numeric constants don't have addresses"; - - PyErr_Format(PyExc_TypeError, - "cannot take the address of '%s' (%s)", - fieldname, reason); - return NULL; - } - else { - PyErr_SetString(PyExc_TypeError, "addressof() first argument must be " - "a cdata struct or union, a pointer to one, or a Lib " - "object"); - return NULL; - } + result = new_simple_cdata(((CDataObject *)arg)->c_data + offset, + (CTypeDescrObject *)z); + Py_DECREF(z); + return result; } -#endif static PyObject *_combine_type_name_l(CTypeDescrObject *ct, size_t extra_text_len) @@ -695,28 +675,26 @@ } static PyMethodDef ffi_methods[] = { - {"__set_types",(PyCFunction)ffi__set_types, METH_VARARGS}, + {"__set_types",(PyCFunction)ffi__set_types, METH_VARARGS}, + {"addressof", (PyCFunction)ffi_addressof, METH_VARARGS, ffi_addressof_doc}, + {"alignof", (PyCFunction)ffi_alignof, METH_O, ffi_alignof_doc}, + {"callback", (PyCFunction)ffi_callback, METH_VARARGS | + METH_KEYWORDS,ffi_callback_doc}, + {"cast", (PyCFunction)ffi_cast, METH_VARARGS, ffi_cast_doc}, #if 0 - {"addressof", (PyCFunction)ffi_addressof, METH_VARARGS}, + {"from_handle",(PyCFunction)ffi_from_handle,METH_O}, + {"gc", (PyCFunction)ffi_gc, METH_VARARGS}, #endif - {"alignof", (PyCFunction)ffi_alignof, METH_O, ffi_alignof_doc}, - {"callback", (PyCFunction)ffi_callback, METH_VARARGS | - METH_KEYWORDS,ffi_callback_doc}, - {"cast", (PyCFunction)ffi_cast, METH_VARARGS, ffi_cast_doc}, + {"getctype", (PyCFunction)ffi_getctype, METH_VARARGS, ffi_getctype_doc}, + {"offsetof", (PyCFunction)ffi_offsetof, METH_VARARGS, ffi_offsetof_doc}, + {"new", (PyCFunction)ffi_new, METH_VARARGS, ffi_new_doc}, #if 0 - {"from_handle",(PyCFunction)ffi_from_handle,METH_O}, - {"gc", (PyCFunction)ffi_gc, METH_VARARGS}, + {"new_handle", (PyCFunction)ffi_new_handle, METH_O}, #endif - {"getctype", (PyCFunction)ffi_getctype, METH_VARARGS, ffi_getctype_doc}, - {"offsetof", (PyCFunction)ffi_offsetof, METH_VARARGS, ffi_offsetof_doc}, - {"new", (PyCFunction)ffi_new, METH_VARARGS, ffi_new_doc}, -#if 0 - {"new_handle", (PyCFunction)ffi_new_handle, METH_O}, -#endif - {"sizeof", (PyCFunction)ffi_sizeof, METH_O, ffi_sizeof_doc}, - {"string", (PyCFunction)ffi_string, METH_VARARGS, ffi_string_doc}, - {"typeof", (PyCFunction)ffi_typeof, METH_O, ffi_typeof_doc}, - {NULL} + {"sizeof", (PyCFunction)ffi_sizeof, METH_O, ffi_sizeof_doc}, + {"string", (PyCFunction)ffi_string, METH_VARARGS, ffi_string_doc}, + {"typeof", (PyCFunction)ffi_typeof, METH_O, ffi_typeof_doc}, + {NULL} }; static PyGetSetDef ffi_getsets[] = { diff --git a/_cffi1/test_ffi_obj.py b/_cffi1/test_ffi_obj.py --- a/_cffi1/test_ffi_obj.py +++ b/_cffi1/test_ffi_obj.py @@ -114,3 +114,10 @@ assert ffi.getctype("int[5]", '*') == "int(*)[5]" assert ffi.getctype("int[5]", '*foo') == "int(*foo)[5]" assert ffi.getctype("int[5]", ' ** foo ') == "int(** foo)[5]" + +def test_addressof(): + ffi = _cffi1_backend.FFI() + a = ffi.new("int[10]") + b = ffi.addressof(a, 5) + b[2] = -123 + assert a[7] == -123 diff --git a/_cffi1/test_recompiler.py b/_cffi1/test_recompiler.py --- a/_cffi1/test_recompiler.py +++ b/_cffi1/test_recompiler.py @@ -235,6 +235,12 @@ # assert ffi.offsetof("struct foo_s", "a") == 0 assert ffi.offsetof("struct foo_s", "b") == 4 + # + py.test.raises(TypeError, ffi.addressof, p) + assert ffi.addressof(p[0]) == p + assert ffi.typeof(ffi.addressof(p[0])) is ffi.typeof("struct foo_s *") + assert ffi.typeof(ffi.addressof(p, "b")) is ffi.typeof("int *") + assert ffi.addressof(p, "b")[0] == p.b def test_verify_exact_field_offset(): ffi = FFI() From noreply at buildbot.pypy.org Sat Apr 25 15:05:51 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 25 Apr 2015 15:05:51 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: ffi.new_handle(), ffi.from_handle() Message-ID: <20150425130551.C40D81C03F7@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1826:e76db3405d4b Date: 2015-04-25 15:04 +0200 http://bitbucket.org/cffi/cffi/changeset/e76db3405d4b/ Log: ffi.new_handle(), ffi.from_handle() diff --git a/_cffi1/ffi_obj.c b/_cffi1/ffi_obj.c --- a/_cffi1/ffi_obj.c +++ b/_cffi1/ffi_obj.c @@ -464,17 +464,21 @@ return res; } -#if 0 -static PyObject *ffi_new_handle(ZefFFIObject *self, PyObject *arg) +PyDoc_STRVAR(ffi_new_handle_doc, +"Return a non-NULL cdata of type 'void *' that contains an opaque\n" +"reference to the argument, which can be any Python object. To cast it\n" +"back to the original object, use from_handle(). You must keep alive\n" +"the cdata object returned by new_handle()!"); + +static PyObject *ffi_new_handle(FFIObject *self, PyObject *arg) { - CTypeDescrObject *ct = ZefNULL->c_type; // CDataObject *cd; cd = (CDataObject *)PyObject_GC_New(CDataObject, &CDataOwningGC_Type); if (cd == NULL) return NULL; - Py_INCREF(ct); - cd->c_type = ct; + Py_INCREF(g_ct_voidp); // + cd->c_type = g_ct_voidp; Py_INCREF(arg); cd->c_data = ((char *)arg) - 42; cd->c_weakreflist = NULL; @@ -482,6 +486,12 @@ return (PyObject *)cd; } +PyDoc_STRVAR(ffi_from_handle_doc, +"Cast a 'void *' back to a Python object. Must be used *only* on the\n" +"pointers returned by new_handle(), and *only* as long as the exact\n" +"cdata object returned by new_handle() is still alive (somewhere else\n" +"in the program). Failure to follow these rules will crash."); + static PyObject *ffi_from_handle(PyObject *self, PyObject *arg) { CTypeDescrObject *ct; @@ -509,6 +519,7 @@ return x; } +#if 0 static PyObject *ffi_gc(ZefFFIObject *self, PyObject *args) { CDataObject *cd; @@ -681,16 +692,14 @@ {"callback", (PyCFunction)ffi_callback, METH_VARARGS | METH_KEYWORDS,ffi_callback_doc}, {"cast", (PyCFunction)ffi_cast, METH_VARARGS, ffi_cast_doc}, + {"from_handle",(PyCFunction)ffi_from_handle,METH_O, ffi_from_handle_doc}, #if 0 - {"from_handle",(PyCFunction)ffi_from_handle,METH_O}, {"gc", (PyCFunction)ffi_gc, METH_VARARGS}, #endif {"getctype", (PyCFunction)ffi_getctype, METH_VARARGS, ffi_getctype_doc}, {"offsetof", (PyCFunction)ffi_offsetof, METH_VARARGS, ffi_offsetof_doc}, {"new", (PyCFunction)ffi_new, METH_VARARGS, ffi_new_doc}, -#if 0 - {"new_handle", (PyCFunction)ffi_new_handle, METH_O}, -#endif + {"new_handle", (PyCFunction)ffi_new_handle, METH_O, ffi_new_handle_doc}, {"sizeof", (PyCFunction)ffi_sizeof, METH_O, ffi_sizeof_doc}, {"string", (PyCFunction)ffi_string, METH_VARARGS, ffi_string_doc}, {"typeof", (PyCFunction)ffi_typeof, METH_O, ffi_typeof_doc}, diff --git a/_cffi1/realize_c_type.c b/_cffi1/realize_c_type.c --- a/_cffi1/realize_c_type.c +++ b/_cffi1/realize_c_type.c @@ -8,6 +8,7 @@ static PyObject *all_primitives[_CFFI__NUM_PRIM]; static PyObject *global_types_dict; +static CTypeDescrObject *g_ct_voidp; static PyObject *build_primitive_type(int num); /* forward */ @@ -39,9 +40,9 @@ Py_DECREF(ct2); return -1; } + g_ct_voidp = (CTypeDescrObject *)ct2; pnull = new_simple_cdata(NULL, (CTypeDescrObject *)ct2); - Py_DECREF(ct2); if (pnull == NULL) return -1; err = PyDict_SetItemString(ffi_type_dict, "NULL", pnull); diff --git a/_cffi1/test_ffi_obj.py b/_cffi1/test_ffi_obj.py --- a/_cffi1/test_ffi_obj.py +++ b/_cffi1/test_ffi_obj.py @@ -121,3 +121,12 @@ b = ffi.addressof(a, 5) b[2] = -123 assert a[7] == -123 + +def test_handle(): + ffi = _cffi1_backend.FFI() + x = [2, 4, 6] + xp = ffi.new_handle(x) + assert ffi.typeof(xp) == ffi.typeof("void *") + assert ffi.from_handle(xp) is x + yp = ffi.new_handle([6, 4, 2]) + assert ffi.from_handle(yp) == [6, 4, 2] From noreply at buildbot.pypy.org Sat Apr 25 15:05:52 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 25 Apr 2015 15:05:52 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: skip some tests Message-ID: <20150425130552.DA18B1C03F7@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1827:d1015f99712f Date: 2015-04-25 15:06 +0200 http://bitbucket.org/cffi/cffi/changeset/d1015f99712f/ Log: skip some tests diff --git a/_cffi1/test_verify1.py b/_cffi1/test_verify1.py --- a/_cffi1/test_verify1.py +++ b/_cffi1/test_verify1.py @@ -1540,6 +1540,7 @@ py.test.raises(TypeError, lib.sum_coord, res2) def test_callback_in_thread(): + py.test.skip("adapt or remove") if sys.platform == 'win32': py.test.skip("pthread only") import os, subprocess, imp @@ -1550,6 +1551,7 @@ assert result == 0 def test_keepalive_lib(): + py.test.skip("adapt or remove") ffi = FFI() ffi.cdef("int foobar(void);") lib = ffi.verify("int foobar(void) { return 42; }") @@ -1563,6 +1565,7 @@ assert func() == 42 def test_keepalive_ffi(): + py.test.skip("adapt or remove") ffi = FFI() ffi.cdef("int foobar(void);") lib = ffi.verify("int foobar(void) { return 42; }") From noreply at buildbot.pypy.org Sat Apr 25 19:58:40 2015 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 25 Apr 2015 19:58:40 +0200 (CEST) Subject: [pypy-commit] pypy default: document merged branch Message-ID: <20150425175840.186841C0228@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76933:98e1cf6b4854 Date: 2015-04-25 20:58 +0300 http://bitbucket.org/pypy/pypy/changeset/98e1cf6b4854/ Log: document merged branch diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -67,3 +67,7 @@ .. branch: object-dtype2 Extend numpy dtypes to allow using objects with associated garbage collection hook + +.. branch: vmprof2 +Add backend support for vmprof - a lightweight statistical profiler - +to linux64, see client at https://vmprof.readthedocs.org From noreply at buildbot.pypy.org Sat Apr 25 21:02:43 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Sat, 25 Apr 2015 21:02:43 +0200 (CEST) Subject: [pypy-commit] pypy can_cast: Add .elsize-dependent checks to dtype.can_cast_to() Message-ID: <20150425190243.A0E371C03F7@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: can_cast Changeset: r76934:6c7224a53fc9 Date: 2015-04-25 20:02 +0100 http://bitbucket.org/pypy/pypy/changeset/6c7224a53fc9/ Log: Add .elsize-dependent checks to dtype.can_cast_to() diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py --- a/pypy/module/micronumpy/descriptor.py +++ b/pypy/module/micronumpy/descriptor.py @@ -40,6 +40,8 @@ return out +_REQ_STRLEN = [0, 3, 5, 10, 10, 20, 20, 20, 20] # data for can_cast_to() + class W_Dtype(W_Root): _immutable_fields_ = [ "itemtype?", "w_box_type", "byteorder?", "names?", "fields?", @@ -94,7 +96,37 @@ return self.itemtype.box_complex(real, imag) def can_cast_to(self, other): - return self.itemtype.can_cast_to(other.itemtype) + result = self.itemtype.can_cast_to(other.itemtype) + if result: + if self.num == NPY.STRING: + if other.num == NPY.STRING: + return self.elsize <= other.elsize + elif other.num == NPY.UNICODE: + return self.elsize * 4 <= other.elsize + elif self.num == NPY.UNICODE and other.num == NPY.UNICODE: + return self.elsize <= other.elsize + elif other.num in (NPY.STRING, NPY.UNICODE): + if other.num == NPY.STRING: + char_size = 1 + else: # NPY.UNICODE + char_size = 4 + if other.elsize == 0: + return True + if self.is_bool(): + return other.elsize >= 5 * char_size + elif self.is_unsigned(): + if self.elsize > 8 or self.elsize < 0: + return False + else: + return (other.elsize >= + _REQ_STRLEN[self.elsize] * char_size) + elif self.is_signed(): + if self.elsize > 8 or self.elsize < 0: + return False + else: + return (other.elsize >= + (_REQ_STRLEN[self.elsize] + 1) * char_size) + return result def coerce(self, space, w_item): return self.itemtype.coerce(space, self, w_item) From noreply at buildbot.pypy.org Sat Apr 25 21:17:14 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Sat, 25 Apr 2015 21:17:14 +0200 (CEST) Subject: [pypy-commit] pypy can_cast: Handle None in np.can_cast() Message-ID: <20150425191715.006941C00F7@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: can_cast Changeset: r76935:ac2f88bfc375 Date: 2015-04-25 20:17 +0100 http://bitbucket.org/pypy/pypy/changeset/ac2f88bfc375/ Log: Handle None in np.can_cast() diff --git a/pypy/module/micronumpy/arrayops.py b/pypy/module/micronumpy/arrayops.py --- a/pypy/module/micronumpy/arrayops.py +++ b/pypy/module/micronumpy/arrayops.py @@ -304,8 +304,12 @@ @unwrap_spec(casting=str) def can_cast(space, w_from, w_totype, casting='safe'): - target = as_dtype(space, w_totype) - origin = as_dtype(space, w_from) # XXX + try: + target = as_dtype(space, w_totype, allow_None=False) + origin = as_dtype(space, w_from, allow_None=False) # XXX + except TypeError: + raise oefmt(space.w_TypeError, + "did not understand one of the types; 'None' not accepted") return space.wrap(can_cast_type(space, origin, target, casting)) kind_ordering = { @@ -331,8 +335,10 @@ return origin.can_cast_to(target) -def as_dtype(space, w_arg): +def as_dtype(space, w_arg, allow_None=True): # roughly equivalent to CNumPy's PyArray_DescrConverter2 + if not allow_None and space.is_none(w_arg): + raise TypeError("Cannot create dtype from None here") if isinstance(w_arg, W_NDimArray): return w_arg.get_dtype() elif isinstance(w_arg, W_GenericBox) or ( From noreply at buildbot.pypy.org Sat Apr 25 22:21:14 2015 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 25 Apr 2015 22:21:14 +0200 (CEST) Subject: [pypy-commit] pypy default: fix so tests pass, even though vmprof is not supported in win32 Message-ID: <20150425202114.09F871C022E@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76936:c0a4171c4d0f Date: 2015-04-25 23:21 +0300 http://bitbucket.org/pypy/pypy/changeset/c0a4171c4d0f/ Log: fix so tests pass, even though vmprof is not supported in win32 diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py --- a/rpython/jit/backend/llsupport/codemap.py +++ b/rpython/jit/backend/llsupport/codemap.py @@ -25,6 +25,10 @@ srcdir = os.path.join(os.path.dirname(__file__), 'src') +libraries = [] +if os.name == 'nt': + libraries.append('Kernel32') + eci = ExternalCompilationInfo(post_include_bits=[""" RPY_EXTERN long pypy_jit_codemap_add(unsigned long addr, unsigned int machine_code_size, @@ -43,7 +47,7 @@ """], separate_module_sources=[ open(os.path.join(srcdir, 'skiplist.c'), 'r').read() + open(os.path.join(srcdir, 'codemap.c'), 'r').read() -], include_dirs=[cdir]) +], include_dirs=[cdir], libraries=libraries) def llexternal(name, args, res): return rffi.llexternal(name, args, res, compilation_info=eci, diff --git a/rpython/jit/backend/llsupport/src/codemap.c b/rpython/jit/backend/llsupport/src/codemap.c --- a/rpython/jit/backend/llsupport/src/codemap.c +++ b/rpython/jit/backend/llsupport/src/codemap.c @@ -15,7 +15,7 @@ else __sync_lock_release(&pypy_codemap_currently_invalid); #else - InterlockedExchange((long volatile *)&pypy_codemap_currently_invalid, + _InterlockedExchange((long volatile *)&pypy_codemap_currently_invalid, (long)value); #endif } From noreply at buildbot.pypy.org Sun Apr 26 04:03:31 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Sun, 26 Apr 2015 04:03:31 +0200 (CEST) Subject: [pypy-commit] pypy can_cast: Add stub implementation for can_cast(, dtype) Message-ID: <20150426020331.D4D101C1207@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: can_cast Changeset: r76937:6aaed87e0b78 Date: 2015-04-26 03:03 +0100 http://bitbucket.org/pypy/pypy/changeset/6aaed87e0b78/ Log: Add stub implementation for can_cast(, dtype) diff --git a/pypy/module/micronumpy/arrayops.py b/pypy/module/micronumpy/arrayops.py --- a/pypy/module/micronumpy/arrayops.py +++ b/pypy/module/micronumpy/arrayops.py @@ -306,7 +306,18 @@ def can_cast(space, w_from, w_totype, casting='safe'): try: target = as_dtype(space, w_totype, allow_None=False) - origin = as_dtype(space, w_from, allow_None=False) # XXX + except TypeError: + raise oefmt(space.w_TypeError, + "did not understand one of the types; 'None' not accepted") + if isinstance(w_from, W_NDimArray): + return space.wrap(can_cast_array(space, w_from, target, casting)) + elif is_scalar_w(space, w_from): + w_scalar = as_scalar(space, w_from) + w_arr = W_NDimArray.from_scalar(space, w_scalar) + return space.wrap(can_cast_array(space, w_arr, target, casting)) + + try: + origin = as_dtype(space, w_from, allow_None=False) except TypeError: raise oefmt(space.w_TypeError, "did not understand one of the types; 'None' not accepted") @@ -334,6 +345,30 @@ else: return origin.can_cast_to(target) +def can_cast_array(space, w_from, target, casting): + origin = w_from.get_dtype() + if w_from.is_scalar(): + return can_cast_scalar( + space, origin, w_from.get_scalar_value(), target, casting) + else: + return can_cast_type(space, origin, target, casting) + +def can_cast_scalar(space, from_type, value, target, casting): + if from_type == target or casting == 'unsafe': + return True + if not from_type.is_number() or casting in ('no', 'equiv'): + return can_cast_type(space, from_type, target, casting) + if not from_type.is_native(): + value = value.descr_byteswap(space) + return can_cast_type(space, from_type, target, casting) # XXX: stub impl + +def is_scalar_w(space, w_arg): + return (isinstance(w_arg, W_GenericBox) or + space.isinstance_w(w_arg, space.w_int) or + space.isinstance_w(w_arg, space.w_float) or + space.isinstance_w(w_arg, space.w_complex) or + space.isinstance_w(w_arg, space.w_long) or + space.isinstance_w(w_arg, space.w_bool)) def as_dtype(space, w_arg, allow_None=True): # roughly equivalent to CNumPy's PyArray_DescrConverter2 @@ -341,13 +376,12 @@ raise TypeError("Cannot create dtype from None here") if isinstance(w_arg, W_NDimArray): return w_arg.get_dtype() - elif isinstance(w_arg, W_GenericBox) or ( - space.isinstance_w(w_arg, space.w_int) or - space.isinstance_w(w_arg, space.w_float) or - space.isinstance_w(w_arg, space.w_complex) or - space.isinstance_w(w_arg, space.w_long) or - space.isinstance_w(w_arg, space.w_bool)): + elif is_scalar_w(space, w_arg): return ufuncs.find_dtype_for_scalar(space, w_arg) else: return space.interp_w(descriptor.W_Dtype, space.call_function(space.gettypefor(descriptor.W_Dtype), w_arg)) + +def as_scalar(space, w_obj): + dtype = ufuncs.find_dtype_for_scalar(space, w_obj) + return dtype.coerce(space, w_obj) diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py --- a/pypy/module/micronumpy/descriptor.py +++ b/pypy/module/micronumpy/descriptor.py @@ -150,6 +150,9 @@ def is_complex(self): return self.kind == NPY.COMPLEXLTR + def is_number(self): + return self.is_int() or self.is_float() or self.is_complex() + def is_str(self): return self.num == NPY.STRING diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -4047,3 +4047,9 @@ raises(TypeError, np.can_cast, 'i4', None) raises(TypeError, np.can_cast, None, 'i4') + + def test_can_cast_scalar(self): + import numpy as np + assert np.can_cast(127, np.int8) + assert not np.can_cast(128, np.int8) + assert np.can_cast(128, np.int16) From noreply at buildbot.pypy.org Sun Apr 26 09:34:05 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 26 Apr 2015 09:34:05 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: tweaks to tests Message-ID: <20150426073405.6D5791C0EC0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1828:9f5ea786ded2 Date: 2015-04-26 09:23 +0200 http://bitbucket.org/cffi/cffi/changeset/9f5ea786ded2/ Log: tweaks to tests diff --git a/_cffi1/support.py b/_cffi1/support.py new file mode 100644 --- /dev/null +++ b/_cffi1/support.py @@ -0,0 +1,6 @@ + +class U(object): + def __add__(self, other): + return eval('u'+repr(other).replace(r'\\u', r'\u') + .replace(r'\\U', r'\U')) +u = U() diff --git a/_cffi1/test_recompiler.py b/_cffi1/test_recompiler.py --- a/_cffi1/test_recompiler.py +++ b/_cffi1/test_recompiler.py @@ -190,7 +190,6 @@ ffi = FFI() ffi.cdef("static double *const FOOBAR;") lib = verify(ffi, 'test_constant_ptr', "#define FOOBAR NULL") - py.test.skip("XXX in-progress:") assert lib.FOOBAR == ffi.NULL assert ffi.typeof(lib.FOOBAR) == ffi.typeof("double *") diff --git a/_cffi1/test_verify1.py b/_cffi1/test_verify1.py --- a/_cffi1/test_verify1.py +++ b/_cffi1/test_verify1.py @@ -1,6 +1,7 @@ import sys, math, py from cffi import FFI, VerificationError, VerificationMissing, model -from _cffi1 import recompiler +from . import recompiler +from .support import * import _cffi_backend lib_m = ['m'] @@ -36,12 +37,6 @@ class FFI_warnings_not_error(FFI): _extra_compile_args = [] -class U(object): - def __add__(self, other): - return eval('u'+repr(other).replace(r'\\u', r'\u') - .replace(r'\\U', r'\U')) -u = U() - def test_missing_function(ffi=None): # uses the FFI hacked above with '-Werror' @@ -1540,7 +1535,7 @@ py.test.raises(TypeError, lib.sum_coord, res2) def test_callback_in_thread(): - py.test.skip("adapt or remove") + py.test.xfail("adapt or remove") if sys.platform == 'win32': py.test.skip("pthread only") import os, subprocess, imp @@ -1551,7 +1546,7 @@ assert result == 0 def test_keepalive_lib(): - py.test.skip("adapt or remove") + py.test.xfail("adapt or remove") ffi = FFI() ffi.cdef("int foobar(void);") lib = ffi.verify("int foobar(void) { return 42; }") @@ -1565,7 +1560,7 @@ assert func() == 42 def test_keepalive_ffi(): - py.test.skip("adapt or remove") + py.test.xfail("adapt or remove") ffi = FFI() ffi.cdef("int foobar(void);") lib = ffi.verify("int foobar(void) { return 42; }") From noreply at buildbot.pypy.org Sun Apr 26 09:34:06 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 26 Apr 2015 09:34:06 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: test and fix Message-ID: <20150426073406.86CB71C0EC0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1829:60463383e5bc Date: 2015-04-26 09:34 +0200 http://bitbucket.org/cffi/cffi/changeset/60463383e5bc/ Log: test and fix diff --git a/_cffi1/realize_c_type.c b/_cffi1/realize_c_type.c --- a/_cffi1/realize_c_type.c +++ b/_cffi1/realize_c_type.c @@ -460,9 +460,13 @@ } PyObject *args = NULL; - if (!PyErr_Occurred()) - args = Py_BuildValue("(sOOO)", e->name, enumerators, + if (!PyErr_Occurred()) { + char *name = alloca(6 + strlen(e->name)); + strcpy(name, "enum "); + strcat(name, e->name); + args = Py_BuildValue("(sOOO)", name, enumerators, enumvalues, basetd); + } Py_DECREF(enumerators); Py_DECREF(enumvalues); if (args == NULL) diff --git a/_cffi1/test_recompiler.py b/_cffi1/test_recompiler.py --- a/_cffi1/test_recompiler.py +++ b/_cffi1/test_recompiler.py @@ -282,6 +282,7 @@ assert lib.B2 == 1 assert ffi.sizeof("enum e1") == ffi.sizeof("long") assert ffi.sizeof("enum e2") == ffi.sizeof("int") + assert repr(ffi.cast("enum e1", 0)) == "" def test_duplicate_enum(): ffi = FFI() @@ -348,6 +349,7 @@ "typedef struct { long b; int hidden, a; } foo_t;") p = ffi.new("foo_t *", {'b': 42}) assert p.b == 42 + assert repr(p).startswith("" # ffi = FFI() ffi.cdef("typedef enum { AA=%d } e1;" % sys.maxint) From noreply at buildbot.pypy.org Sun Apr 26 10:13:37 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 26 Apr 2015 10:13:37 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: give the expected name to $struct and $enums Message-ID: <20150426081337.B718A1C0359@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1830:c92ee3ee4642 Date: 2015-04-26 09:58 +0200 http://bitbucket.org/cffi/cffi/changeset/c92ee3ee4642/ Log: give the expected name to $struct and $enums diff --git a/_cffi1/realize_c_type.c b/_cffi1/realize_c_type.c --- a/_cffi1/realize_c_type.c +++ b/_cffi1/realize_c_type.c @@ -288,6 +288,38 @@ } } +static void _realize_name(char *target, const char *prefix, const char *srcname) +{ + /* "xyz" => "struct xyz" + "$xyz" => "xyz" + */ + if (srcname[0] == '$' && srcname[1] != '$') { + strcpy(target, &srcname[1]); + } + else { + strcpy(target, prefix); + strcat(target, srcname); + } +} + +static void _unrealize_name(char *target, const char *srcname) +{ + /* reverse of _realize_name() */ + if (strncmp(srcname, "struct ", 7) == 0) { + strcpy(target, &srcname[7]); + } + else if (strncmp(srcname, "union ", 6) == 0) { + strcpy(target, &srcname[6]); + } + else if (strncmp(srcname, "enum ", 5) == 0) { + strcpy(target, &srcname[5]); + } + else { + strcpy(target, "$"); + strcat(target, srcname); + } +} + static PyObject * _realize_c_type_or_func(builder_c_t *builder, _cffi_opcode_t opcodes[], int index) @@ -354,17 +386,11 @@ Py_INCREF(x); } else { - int flags; + int flags = (s->flags & CT_UNION) ? CT_UNION : CT_STRUCT; char *name = alloca(8 + strlen(s->name)); - if (s->flags & CT_UNION) { - strcpy(name, "union "); - flags = CT_UNION; - } - else { - strcpy(name, "struct "); - flags = CT_STRUCT; - } - strcat(name, s->name); + _realize_name(name, + (s->flags & CT_UNION) ? "union " : "struct ", + s->name); x = new_struct_or_union_type(name, flags); CTypeDescrObject *ct = NULL; @@ -462,8 +488,7 @@ PyObject *args = NULL; if (!PyErr_Occurred()) { char *name = alloca(6 + strlen(e->name)); - strcpy(name, "enum "); - strcat(name, e->name); + _realize_name(name, "enum ", e->name); args = Py_BuildValue("(sOOO)", name, enumerators, enumvalues, basetd); } @@ -577,11 +602,8 @@ builder_c_t *builder = ct->ct_extra; assert(builder != NULL); - char *p = ct->ct_name; - if (memcmp(p, "struct ", 7) == 0) - p += 7; - else if (memcmp(p, "union ", 6) == 0) - p += 6; + char *p = alloca(2 + strlen(ct->ct_name)); + _unrealize_name(p, ct->ct_name); int n = search_in_struct_unions(&builder->ctx, p, strlen(p)); if (n < 0) diff --git a/_cffi1/test_recompiler.py b/_cffi1/test_recompiler.py --- a/_cffi1/test_recompiler.py +++ b/_cffi1/test_recompiler.py @@ -349,7 +349,7 @@ "typedef struct { long b; int hidden, a; } foo_t;") p = ffi.new("foo_t *", {'b': 42}) assert p.b == 42 - assert repr(p).startswith("" + assert repr(ffi.cast("e1", 2)) == "" # ffi = FFI() ffi.cdef("typedef enum { AA=%d } e1;" % sys.maxint) From noreply at buildbot.pypy.org Sun Apr 26 10:13:38 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 26 Apr 2015 10:13:38 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: in-progress: import backend_tests Message-ID: <20150426081338.EDE6C1C0359@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1831:48276f27dd98 Date: 2015-04-26 10:14 +0200 http://bitbucket.org/cffi/cffi/changeset/48276f27dd98/ Log: in-progress: import backend_tests diff --git a/testing/backend_tests.py b/_cffi1/test_new_ffi_1.py copy from testing/backend_tests.py copy to _cffi1/test_new_ffi_1.py --- a/testing/backend_tests.py +++ b/_cffi1/test_new_ffi_1.py @@ -1,8 +1,11 @@ import py -import platform +import platform, imp import sys, ctypes -from cffi import FFI, CDefError, FFIError -from testing.support import * +import cffi +from cffi import CDefError, FFIError +from .udir import udir +from .recompiler import recompile +from .support import * SIZE_OF_INT = ctypes.sizeof(ctypes.c_int) SIZE_OF_LONG = ctypes.sizeof(ctypes.c_long) @@ -11,10 +14,75 @@ SIZE_OF_WCHAR = ctypes.sizeof(ctypes.c_wchar) -class BackendTests: +def setup_module(): + global ffi + ffi1 = cffi.FFI() + DEFS = r""" + struct repr { short a, b, c; }; + struct simple { int a; short b, c; }; + struct array { int a[2]; char b[3]; }; + struct recursive { int value; struct recursive *next; }; + union simple_u { int a; short b, c; }; + union init_u { char a; int b; }; + struct four_s { int a; short b, c, d; }; + union four_u { int a; short b, c, d; }; + struct string { const char *name; }; + struct ustring { const wchar_t *name; }; + struct voidp { void *p; int *q; short *r; }; + struct ab { int a, b; }; + struct abc { int a, b, c; }; + + enum foq { A0, B0, CC0, D0 }; + enum bar { A1, B1=-2, CC1, D1, E1 }; + enum baz { A2=0x1000, B2=0x2000 }; + enum foo2 { A3, B3, C3, D3 }; + struct bar_with_e { enum foo2 e; }; + enum noncont { A4, B4=42, C4 }; + enum etypes {A5='!', B5='\'', C5=0x10, D5=010, E5=- 0x10, F5=-010}; + typedef enum { Value0 = 0 } e_t, *pe_t; + enum e_noninj { AA3=0, BB3=0, CC3=0, DD3=0 }; + enum e_prev { AA4, BB4=2, CC4=4, DD4=BB4, EE4, FF4=CC4, GG4=FF4 }; + + struct nesting { struct abc d, e; }; + struct array2 { int a, b; int c[99]; }; + struct align { char a; short b; char c; }; + struct bitfield { int a:10, b:20, c:3; }; + typedef enum { AA2, BB2, CC2 } foo_e_t; + typedef struct { foo_e_t f:2; } bfenum_t; + typedef struct { int a; } anon_foo_t; + typedef struct { char b, c; } anon_bar_t; + typedef struct named_foo_s { int a; } named_foo_t, *named_foo_p; + typedef struct { int a; } unnamed_foo_t, *unnamed_foo_p; + struct nonpacked { char a; int b; }; + struct array0 { int len; short data[0]; }; + + struct nested_anon { + struct { int a, b; }; + union { int c, d; }; + }; + union nested_anon_u { + struct { int a, b; }; + union { int c, d; }; + }; + struct abc50 { int a, b; int c[50]; }; + """ + DEFS_PACKED = """ + struct is_packed { char a; int b; } /*here*/; + """ + CCODE = DEFS + DEFS_PACKED.replace('/*here*/', '__attribute__((packed))') + + ffi1.cdef(DEFS) + ffi1.cdef(DEFS_PACKED, packed=True) + + outputfilename = recompile(ffi1, "test_old_ffi1", CCODE, + tmpdir=str(udir)) + module = imp.load_dynamic("test_old_ffi1", outputfilename) + ffi = module.ffi + + +class TestOldFFI1: def test_integer_ranges(self): - ffi = FFI(backend=self.Backend()) for (c_type, size) in [('char', 1), ('short', 2), ('short int', 2), @@ -34,7 +102,6 @@ self._test_int_type(ffi, c_decl, size, unsigned) def test_fixedsize_int(self): - ffi = FFI(backend=self.Backend()) for size in [1, 2, 4, 8]: self._test_int_type(ffi, 'int%d_t' % (8*size), size, False) self._test_int_type(ffi, 'uint%d_t' % (8*size), size, True) @@ -79,12 +146,10 @@ assert ffi.new(c_decl_ptr, long(max))[0] == max def test_new_unsupported_type(self): - ffi = FFI(backend=self.Backend()) e = py.test.raises(TypeError, ffi.new, "int") assert str(e.value) == "expected a pointer or array ctype, got 'int'" def test_new_single_integer(self): - ffi = FFI(backend=self.Backend()) p = ffi.new("int *") # similar to ffi.new("int[1]") assert p[0] == 0 p[0] = -123 @@ -94,14 +159,12 @@ assert repr(p) == "" % SIZE_OF_INT def test_new_array_no_arg(self): - ffi = FFI(backend=self.Backend()) p = ffi.new("int[10]") # the object was zero-initialized: for i in range(10): assert p[i] == 0 def test_array_indexing(self): - ffi = FFI(backend=self.Backend()) p = ffi.new("int[10]") p[0] = 42 p[9] = 43 @@ -113,7 +176,6 @@ py.test.raises(IndexError, "p[-1] = 44") def test_new_array_args(self): - ffi = FFI(backend=self.Backend()) # this tries to be closer to C: where we say "int x[5] = {10, 20, ..}" # then here we must enclose the items in a list p = ffi.new("int[5]", [10, 20, 30, 40, 50]) @@ -132,7 +194,6 @@ assert repr(p) == "" % (4*SIZE_OF_INT) def test_new_array_varsize(self): - ffi = FFI(backend=self.Backend()) p = ffi.new("int[]", 10) # a single integer is the length assert p[9] == 0 py.test.raises(IndexError, "p[10]") @@ -151,7 +212,6 @@ assert repr(p) == "" def test_pointer_init(self): - ffi = FFI(backend=self.Backend()) n = ffi.new("int *", 24) a = ffi.new("int *[10]", [ffi.NULL, ffi.NULL, n, n, ffi.NULL]) for i in range(10): @@ -160,14 +220,12 @@ assert a[2] == a[3] == n def test_cannot_cast(self): - ffi = FFI(backend=self.Backend()) a = ffi.new("short int[10]") e = py.test.raises(TypeError, ffi.new, "long int **", a) msg = str(e.value) assert "'short[10]'" in msg and "'long *'" in msg def test_new_pointer_to_array(self): - ffi = FFI(backend=self.Backend()) a = ffi.new("int[4]", [100, 102, 104, 106]) p = ffi.new("int **", a) assert p[0] == ffi.cast("int *", a) @@ -180,7 +238,6 @@ # keepalive: a def test_pointer_direct(self): - ffi = FFI(backend=self.Backend()) p = ffi.cast("int*", 0) assert p is not None assert bool(p) is False @@ -196,9 +253,7 @@ assert p[1] == 456 def test_repr(self): - typerepr = self.TypeRepr - ffi = FFI(backend=self.Backend()) - ffi.cdef("struct foo { short a, b, c; };") + typerepr = "" p = ffi.cast("short unsigned int", 0) assert repr(p) == "" assert repr(ffi.typeof(p)) == typerepr % "unsigned short" @@ -222,10 +277,10 @@ assert repr(p) == "" % ( 6*SIZE_OF_PTR) assert repr(ffi.typeof(p)) == typerepr % "int *[2][3]" - p = ffi.new("struct foo *") - assert repr(p) == "" % ( + p = ffi.new("struct repr *") + assert repr(p) == "" % ( 3*SIZE_OF_SHORT) - assert repr(ffi.typeof(p)) == typerepr % "struct foo *" + assert repr(ffi.typeof(p)) == typerepr % "struct repr *" # q = ffi.cast("short", -123) assert repr(q) == "" @@ -238,17 +293,16 @@ q = ffi.cast("int*", p) assert repr(q).startswith("" % ( + assert repr(s) == "" % ( SIZE_OF_INT + 2 * SIZE_OF_SHORT) # - py.test.raises(ValueError, ffi.new, "struct foo*", [1, 2, 3, 4]) + py.test.raises(ValueError, ffi.new, "struct simple*", [1, 2, 3, 4]) def test_constructor_struct_from_dict(self): - ffi = FFI(backend=self.Backend()) - ffi.cdef("struct foo { int a; short b, c; };") - s = ffi.new("struct foo*", {'b': 123, 'c': 456}) + s = ffi.new("struct simple*", {'b': 123, 'c': 456}) assert s.a == 0 assert s.b == 123 assert s.c == 456 - py.test.raises(KeyError, ffi.new, "struct foo*", {'d': 456}) + py.test.raises(KeyError, ffi.new, "struct simple*", {'d': 456}) def test_struct_pointer(self): - ffi = FFI(backend=self.Backend()) - ffi.cdef("struct foo { int a; short b, c; };") - s = ffi.new("struct foo*") + s = ffi.new("struct simple*") assert s[0].a == s[0].b == s[0].c == 0 s[0].b = -23 assert s[0].b == s.b == -23 @@ -444,18 +485,16 @@ py.test.raises(IndexError, "s[1]") def test_struct_opaque(self): - ffi = FFI(backend=self.Backend()) - py.test.raises(TypeError, ffi.new, "struct baz*") - p = ffi.new("struct baz **") # this works - assert p[0] == ffi.NULL + py.test.raises(ffi.error, ffi.new, "struct baz*") + # should 'ffi.new("struct baz **") work? it used to, but it was + # not particularly useful... + py.test.raises(ffi.error, ffi.new, "struct baz**") def test_pointer_to_struct(self): - ffi = FFI(backend=self.Backend()) - ffi.cdef("struct foo { int a; short b, c; };") - s = ffi.new("struct foo *") + s = ffi.new("struct simple *") s.a = -42 assert s[0].a == -42 - p = ffi.new("struct foo **", s) + p = ffi.new("struct simple **", s) assert p[0].a == -42 assert p[0][0].a == -42 p[0].a = -43 @@ -472,9 +511,7 @@ assert p[0][0].a == -46 def test_constructor_struct_of_array(self): - ffi = FFI(backend=self.Backend()) - ffi.cdef("struct foo { int a[2]; char b[3]; };") - s = ffi.new("struct foo *", [[10, 11], [b'a', b'b', b'c']]) + s = ffi.new("struct array *", [[10, 11], [b'a', b'b', b'c']]) assert s.a[1] == 11 assert s.b[2] == b'c' s.b[1] = b'X' @@ -483,10 +520,8 @@ assert s.b[2] == b'c' def test_recursive_struct(self): - ffi = FFI(backend=self.Backend()) - ffi.cdef("struct foo { int value; struct foo *next; };") - s = ffi.new("struct foo*") - t = ffi.new("struct foo*") + s = ffi.new("struct recursive*") + t = ffi.new("struct recursive*") s.value = 123 s.next = t t.value = 456 @@ -494,60 +529,51 @@ assert s.next.value == 456 def test_union_simple(self): - ffi = FFI(backend=self.Backend()) - ffi.cdef("union foo { int a; short b, c; };") - u = ffi.new("union foo*") + u = ffi.new("union simple_u*") assert u.a == u.b == u.c == 0 u.b = -23 assert u.b == -23 assert u.a != 0 py.test.raises(OverflowError, "u.b = 32768") # - u = ffi.new("union foo*", [-2]) + u = ffi.new("union simple_u*", [-2]) assert u.a == -2 py.test.raises((AttributeError, TypeError), "del u.a") - assert repr(u) == "" % SIZE_OF_INT + assert repr(u) == "" % ( + SIZE_OF_INT,) def test_union_opaque(self): - ffi = FFI(backend=self.Backend()) - py.test.raises(TypeError, ffi.new, "union baz *") - u = ffi.new("union baz **") # this works - assert u[0] == ffi.NULL + py.test.raises(ffi.error, ffi.new, "union baz*") + # should 'ffi.new("union baz **") work? it used to, but it was + # not particularly useful... + py.test.raises(ffi.error, ffi.new, "union baz**") def test_union_initializer(self): - ffi = FFI(backend=self.Backend()) - ffi.cdef("union foo { char a; int b; };") - py.test.raises(TypeError, ffi.new, "union foo*", b'A') - py.test.raises(TypeError, ffi.new, "union foo*", 5) - py.test.raises(ValueError, ffi.new, "union foo*", [b'A', 5]) - u = ffi.new("union foo*", [b'A']) + py.test.raises(TypeError, ffi.new, "union init_u*", b'A') + py.test.raises(TypeError, ffi.new, "union init_u*", 5) + py.test.raises(ValueError, ffi.new, "union init_u*", [b'A', 5]) + u = ffi.new("union init_u*", [b'A']) assert u.a == b'A' - py.test.raises(TypeError, ffi.new, "union foo*", [1005]) - u = ffi.new("union foo*", {'b': 12345}) + py.test.raises(TypeError, ffi.new, "union init_u*", [1005]) + u = ffi.new("union init_u*", {'b': 12345}) assert u.b == 12345 - u = ffi.new("union foo*", []) + u = ffi.new("union init_u*", []) assert u.a == b'\x00' assert u.b == 0 def test_sizeof_type(self): - ffi = FFI(backend=self.Backend()) - ffi.cdef(""" - struct foo { int a; short b, c, d; }; - union foo { int a; short b, c, d; }; - """) for c_type, expected_size in [ ('char', 1), ('unsigned int', 4), ('char *', SIZE_OF_PTR), ('int[5]', 20), - ('struct foo', 12), - ('union foo', 4), + ('struct four_s', 12), + ('union four_u', 4), ]: size = ffi.sizeof(c_type) assert size == expected_size, (size, expected_size, ctype) def test_sizeof_cdata(self): - ffi = FFI(backend=self.Backend()) assert ffi.sizeof(ffi.new("short*")) == SIZE_OF_PTR assert ffi.sizeof(ffi.cast("short", 123)) == SIZE_OF_SHORT # @@ -556,7 +582,6 @@ assert ffi.sizeof(a) == 5 * SIZE_OF_INT def test_string_from_char_pointer(self): - ffi = FFI(backend=self.Backend()) x = ffi.new("char*", b"x") assert str(x) == repr(x) assert ffi.string(x) == b"x" @@ -564,7 +589,6 @@ py.test.raises(TypeError, ffi.new, "char*", unicode("foo")) def test_unicode_from_wchar_pointer(self): - ffi = FFI(backend=self.Backend()) self.check_wchar_t(ffi) x = ffi.new("wchar_t*", u+"x") assert unicode(x) == unicode(repr(x)) @@ -572,7 +596,6 @@ assert ffi.string(ffi.new("wchar_t*", u+"\x00")) == u+"" def test_string_from_char_array(self): - ffi = FFI(backend=self.Backend()) p = ffi.new("char[]", b"hello.") p[5] = b'!' assert ffi.string(p) == b"hello!" @@ -589,7 +612,6 @@ assert ffi.string(p) == b'hello' def test_string_from_wchar_array(self): - ffi = FFI(backend=self.Backend()) self.check_wchar_t(ffi) assert ffi.string(ffi.cast("wchar_t", "x")) == u+"x" assert ffi.string(ffi.cast("wchar_t", u+"x")) == u+"x" @@ -616,11 +638,9 @@ assert ffi.string(p, 2) == u+'he' def test_fetch_const_char_p_field(self): - # 'const' is ignored so far - ffi = FFI(backend=self.Backend()) - ffi.cdef("struct foo { const char *name; };") + # 'const' is ignored so far, in the declaration of 'struct string' t = ffi.new("const char[]", b"testing") - s = ffi.new("struct foo*", [t]) + s = ffi.new("struct string*", [t]) assert type(s.name) not in (bytes, str, unicode) assert ffi.string(s.name) == b"testing" py.test.raises(TypeError, "s.name = None") @@ -629,18 +649,15 @@ def test_fetch_const_wchar_p_field(self): # 'const' is ignored so far - ffi = FFI(backend=self.Backend()) self.check_wchar_t(ffi) - ffi.cdef("struct foo { const wchar_t *name; };") t = ffi.new("const wchar_t[]", u+"testing") - s = ffi.new("struct foo*", [t]) + s = ffi.new("struct ustring*", [t]) assert type(s.name) not in (bytes, str, unicode) assert ffi.string(s.name) == u+"testing" s.name = ffi.NULL assert s.name == ffi.NULL def test_voidp(self): - ffi = FFI(backend=self.Backend()) py.test.raises(TypeError, ffi.new, "void*") p = ffi.new("void **") assert p[0] == ffi.NULL @@ -650,8 +667,7 @@ py.test.raises(TypeError, "vp[0]") py.test.raises(TypeError, ffi.new, "short **", a) # - ffi.cdef("struct foo { void *p; int *q; short *r; };") - s = ffi.new("struct foo *") + s = ffi.new("struct voidp *") s.p = a # works s.q = a # works py.test.raises(TypeError, "s.r = a") # fails @@ -661,7 +677,6 @@ py.test.raises(TypeError, "s.r = b") # fails def test_functionptr_simple(self): - ffi = FFI(backend=self.Backend()) py.test.raises(TypeError, ffi.callback, "int(*)(int)", 0) def cb(n): return n + 1 @@ -686,12 +701,10 @@ assert res == 46 def test_functionptr_advanced(self): - ffi = FFI(backend=self.Backend()) t = ffi.typeof("int(*(*)(int))(int)") - assert repr(t) == self.TypeRepr % "int(*(*)(int))(int)" + assert repr(t) == "" % "int(*(*)(int))(int)" def test_functionptr_voidptr_return(self): - ffi = FFI(backend=self.Backend()) def cb(): return ffi.NULL p = ffi.callback("void*(*)()", cb) @@ -707,7 +720,6 @@ assert res == void_ptr def test_functionptr_intptr_return(self): - ffi = FFI(backend=self.Backend()) def cb(): return ffi.NULL p = ffi.callback("int*(*)()", cb) @@ -729,7 +741,6 @@ assert res == int_array_ptr def test_functionptr_void_return(self): - ffi = FFI(backend=self.Backend()) def foo(): pass foo_cb = ffi.callback("void foo()", foo) @@ -737,7 +748,6 @@ assert result is None def test_char_cast(self): - ffi = FFI(backend=self.Backend()) p = ffi.cast("int", b'\x01') assert ffi.typeof(p) is ffi.typeof("int") assert int(p) == 1 @@ -749,7 +759,6 @@ assert int(p) == 0x81 def test_wchar_cast(self): - ffi = FFI(backend=self.Backend()) self.check_wchar_t(ffi) p = ffi.cast("int", ffi.cast("wchar_t", u+'\u1234')) assert int(p) == 0x1234 @@ -764,7 +773,6 @@ assert int(p) == 0x1234 def test_cast_array_to_charp(self): - ffi = FFI(backend=self.Backend()) a = ffi.new("short int[]", [0x1234, 0x5678]) p = ffi.cast("char*", a) data = b''.join([p[i] for i in range(4)]) @@ -774,7 +782,6 @@ assert data == b'\x12\x34\x56\x78' def test_cast_between_pointers(self): - ffi = FFI(backend=self.Backend()) a = ffi.new("short int[]", [0x1234, 0x5678]) p = ffi.cast("short*", a) p2 = ffi.cast("int*", p) @@ -786,7 +793,6 @@ assert data == b'\x12\x34\x56\x78' def test_cast_pointer_and_int(self): - ffi = FFI(backend=self.Backend()) a = ffi.new("short int[]", [0x1234, 0x5678]) l1 = ffi.cast("intptr_t", a) p = ffi.cast("short*", a) @@ -798,7 +804,6 @@ assert int(ffi.cast("intptr_t", ffi.NULL)) == 0 def test_cast_functionptr_and_int(self): - ffi = FFI(backend=self.Backend()) def cb(n): return n + 1 a = ffi.callback("int(*)(int)", cb) @@ -810,7 +815,6 @@ assert hash(a) == hash(b) def test_callback_crash(self): - ffi = FFI(backend=self.Backend()) def cb(n): raise Exception a = ffi.callback("int(*)(int)", cb, error=42) @@ -818,19 +822,15 @@ assert res == 42 def test_structptr_argument(self): - ffi = FFI(backend=self.Backend()) - ffi.cdef("struct foo_s { int a, b; };") def cb(p): return p[0].a * 1000 + p[0].b * 100 + p[1].a * 10 + p[1].b - a = ffi.callback("int(*)(struct foo_s[])", cb) + a = ffi.callback("int(*)(struct ab[])", cb) res = a([[5, 6], {'a': 7, 'b': 8}]) assert res == 5678 res = a([[5], {'b': 8}]) assert res == 5008 def test_array_argument_as_list(self): - ffi = FFI(backend=self.Backend()) - ffi.cdef("struct foo_s { int a, b; };") seen = [] def cb(argv): seen.append(ffi.string(argv[0])) @@ -840,7 +840,6 @@ assert seen == [b"foobar", b"baz"] def test_cast_float(self): - ffi = FFI(backend=self.Backend()) a = ffi.cast("float", 12) assert float(a) == 12.0 a = ffi.cast("float", 12.5) @@ -864,31 +863,30 @@ assert ffi.string(a) == b"B" def test_enum(self): - ffi = FFI(backend=self.Backend()) - ffi.cdef("enum foo { A0, B0, CC0, D0 };") - assert ffi.string(ffi.cast("enum foo", 0)) == "A0" - assert ffi.string(ffi.cast("enum foo", 2)) == "CC0" - assert ffi.string(ffi.cast("enum foo", 3)) == "D0" - assert ffi.string(ffi.cast("enum foo", 4)) == "4" - ffi.cdef("enum bar { A1, B1=-2, CC1, D1, E1 };") + # enum foq { A0, B0, CC0, D0 }; + assert ffi.string(ffi.cast("enum foq", 0)) == "A0" + assert ffi.string(ffi.cast("enum foq", 2)) == "CC0" + assert ffi.string(ffi.cast("enum foq", 3)) == "D0" + assert ffi.string(ffi.cast("enum foq", 4)) == "4" + # enum bar { A1, B1=-2, CC1, D1, E1 }; assert ffi.string(ffi.cast("enum bar", 0)) == "A1" assert ffi.string(ffi.cast("enum bar", -2)) == "B1" assert ffi.string(ffi.cast("enum bar", -1)) == "CC1" assert ffi.string(ffi.cast("enum bar", 1)) == "E1" assert ffi.cast("enum bar", -2) != ffi.cast("enum bar", -2) - assert ffi.cast("enum foo", 0) != ffi.cast("enum bar", 0) + assert ffi.cast("enum foq", 0) != ffi.cast("enum bar", 0) assert ffi.cast("enum bar", 0) != ffi.cast("int", 0) assert repr(ffi.cast("enum bar", -1)) == "" - assert repr(ffi.cast("enum foo", -1)) == ( # enums are unsigned, if - "") # they contain no neg value - ffi.cdef("enum baz { A2=0x1000, B2=0x2000 };") + assert repr(ffi.cast("enum foq", -1)) == ( # enums are unsigned, if + "") # they contain no neg value + # enum baz { A2=0x1000, B2=0x2000 }; assert ffi.string(ffi.cast("enum baz", 0x1000)) == "A2" assert ffi.string(ffi.cast("enum baz", 0x2000)) == "B2" def test_enum_in_struct(self): - ffi = FFI(backend=self.Backend()) - ffi.cdef("enum foo { A, B, C, D }; struct bar { enum foo e; };") - s = ffi.new("struct bar *") + # enum foo2 { A3, B3, C3, D3 }; + # struct bar_with_e { enum foo2 e; }; + s = ffi.new("struct bar_with_e *") s.e = 0 assert s.e == 0 s.e = 3 @@ -897,39 +895,35 @@ s[0].e = 2 assert s.e == 2 assert s[0].e == 2 - s.e = ffi.cast("enum foo", -1) + s.e = ffi.cast("enum foo2", -1) assert s.e == 4294967295 assert s[0].e == 4294967295 s.e = s.e - py.test.raises(TypeError, "s.e = 'B'") + py.test.raises(TypeError, "s.e = 'B3'") py.test.raises(TypeError, "s.e = '2'") py.test.raises(TypeError, "s.e = '#2'") py.test.raises(TypeError, "s.e = '#7'") def test_enum_non_contiguous(self): - ffi = FFI(backend=self.Backend()) - ffi.cdef("enum foo { A, B=42, C };") - assert ffi.string(ffi.cast("enum foo", 0)) == "A" - assert ffi.string(ffi.cast("enum foo", 42)) == "B" - assert ffi.string(ffi.cast("enum foo", 43)) == "C" - invalid_value = ffi.cast("enum foo", 2) + # enum noncont { A4, B4=42, C4 }; + assert ffi.string(ffi.cast("enum noncont", 0)) == "A4" + assert ffi.string(ffi.cast("enum noncont", 42)) == "B4" + assert ffi.string(ffi.cast("enum noncont", 43)) == "C4" + invalid_value = ffi.cast("enum noncont", 2) assert int(invalid_value) == 2 assert ffi.string(invalid_value) == "2" def test_enum_char_hex_oct(self): - ffi = FFI(backend=self.Backend()) - ffi.cdef(r"enum foo{A='!', B='\'', C=0x10, D=010, E=- 0x10, F=-010};") - assert ffi.string(ffi.cast("enum foo", ord('!'))) == "A" - assert ffi.string(ffi.cast("enum foo", ord("'"))) == "B" - assert ffi.string(ffi.cast("enum foo", 16)) == "C" - assert ffi.string(ffi.cast("enum foo", 8)) == "D" - assert ffi.string(ffi.cast("enum foo", -16)) == "E" - assert ffi.string(ffi.cast("enum foo", -8)) == "F" + # enum etypes {A5='!', B5='\'', C5=0x10, D5=010, E5=- 0x10, F5=-010}; + assert ffi.string(ffi.cast("enum etypes", ord('!'))) == "A5" + assert ffi.string(ffi.cast("enum etypes", ord("'"))) == "B5" + assert ffi.string(ffi.cast("enum etypes", 16)) == "C5" + assert ffi.string(ffi.cast("enum etypes", 8)) == "D5" + assert ffi.string(ffi.cast("enum etypes", -16)) == "E5" + assert ffi.string(ffi.cast("enum etypes", -8)) == "F5" def test_array_of_struct(self): - ffi = FFI(backend=self.Backend()) - ffi.cdef("struct foo { int a, b; };") - s = ffi.new("struct foo[1]") + s = ffi.new("struct ab[1]") py.test.raises(AttributeError, 's.b') py.test.raises(AttributeError, 's.b = 412') s[0].b = 412 @@ -937,12 +931,10 @@ py.test.raises(IndexError, 's[1]') def test_pointer_to_array(self): - ffi = FFI(backend=self.Backend()) p = ffi.new("int(**)[5]") assert repr(p) == "" % SIZE_OF_PTR def test_iterate_array(self): - ffi = FFI(backend=self.Backend()) a = ffi.new("char[]", b"hello") assert list(a) == [b"h", b"e", b"l", b"l", b"o", b"\0"] assert list(iter(a)) == [b"h", b"e", b"l", b"l", b"o", b"\0"] @@ -953,43 +945,37 @@ py.test.raises(TypeError, list, ffi.new("int *")) def test_offsetof(self): - ffi = FFI(backend=self.Backend()) - ffi.cdef("struct foo { int a, b, c; };") - assert ffi.offsetof("struct foo", "a") == 0 - assert ffi.offsetof("struct foo", "b") == 4 - assert ffi.offsetof("struct foo", "c") == 8 + # struct abc { int a, b, c; }; + assert ffi.offsetof("struct abc", "a") == 0 + assert ffi.offsetof("struct abc", "b") == 4 + assert ffi.offsetof("struct abc", "c") == 8 def test_offsetof_nested(self): - ffi = FFI(backend=self.Backend()) - ffi.cdef("struct foo { int a, b, c; };" - "struct bar { struct foo d, e; };") - assert ffi.offsetof("struct bar", "e") == 12 - py.test.raises(KeyError, ffi.offsetof, "struct bar", "e.a") - assert ffi.offsetof("struct bar", "e", "a") == 12 - assert ffi.offsetof("struct bar", "e", "b") == 16 - assert ffi.offsetof("struct bar", "e", "c") == 20 + # struct nesting { struct abc d, e; }; + assert ffi.offsetof("struct nesting", "e") == 12 + py.test.raises(KeyError, ffi.offsetof, "struct nesting", "e.a") + assert ffi.offsetof("struct nesting", "e", "a") == 12 + assert ffi.offsetof("struct nesting", "e", "b") == 16 + assert ffi.offsetof("struct nesting", "e", "c") == 20 def test_offsetof_array(self): - ffi = FFI(backend=self.Backend()) assert ffi.offsetof("int[]", 51) == 51 * ffi.sizeof("int") assert ffi.offsetof("int *", 51) == 51 * ffi.sizeof("int") - ffi.cdef("struct bar { int a, b; int c[99]; };") - assert ffi.offsetof("struct bar", "c") == 2 * ffi.sizeof("int") - assert ffi.offsetof("struct bar", "c", 0) == 2 * ffi.sizeof("int") - assert ffi.offsetof("struct bar", "c", 51) == 53 * ffi.sizeof("int") + # struct array2 { int a, b; int c[99]; }; + assert ffi.offsetof("struct array2", "c") == 2 * ffi.sizeof("int") + assert ffi.offsetof("struct array2", "c", 0) == 2 * ffi.sizeof("int") + assert ffi.offsetof("struct array2", "c", 51) == 53 * ffi.sizeof("int") def test_alignof(self): - ffi = FFI(backend=self.Backend()) - ffi.cdef("struct foo { char a; short b; char c; };") + # struct align { char a; short b; char c; }; assert ffi.alignof("int") == 4 assert ffi.alignof("double") in (4, 8) - assert ffi.alignof("struct foo") == 2 + assert ffi.alignof("struct align") == 2 def test_bitfield(self): - ffi = FFI(backend=self.Backend()) - ffi.cdef("struct foo { int a:10, b:20, c:3; };") - assert ffi.sizeof("struct foo") == 8 - s = ffi.new("struct foo *") + # struct bitfield { int a:10, b:20, c:3; }; + assert ffi.sizeof("struct bitfield") == 8 + s = ffi.new("struct bitfield *") s.a = 511 py.test.raises(OverflowError, "s.a = 512") py.test.raises(OverflowError, "s[0].a = 512") @@ -1006,35 +992,31 @@ assert s.c == -4 def test_bitfield_enum(self): - ffi = FFI(backend=self.Backend()) - ffi.cdef(""" - typedef enum { AA, BB, CC } foo_e; - typedef struct { foo_e f:2; } foo_s; - """) - s = ffi.new("foo_s *") + # typedef enum { AA1, BB1, CC1 } foo_e_t; + # typedef struct { foo_e_t f:2; } bfenum_t; + s = ffi.new("bfenum_t *") s.f = 2 assert s.f == 2 def test_anonymous_struct(self): - ffi = FFI(backend=self.Backend()) - ffi.cdef("typedef struct { int a; } foo_t;") - ffi.cdef("typedef struct { char b, c; } bar_t;") - f = ffi.new("foo_t *", [12345]) - b = ffi.new("bar_t *", [b"B", b"C"]) + # typedef struct { int a; } anon_foo_t; + # typedef struct { char b, c; } anon_bar_t; + f = ffi.new("anon_foo_t *", [12345]) + b = ffi.new("anon_bar_t *", [b"B", b"C"]) assert f.a == 12345 assert b.b == b"B" assert b.c == b"C" - assert repr(b).startswith(" Author: Armin Rigo Branch: cffi-1.0 Changeset: r1832:fd16324ee375 Date: 2015-04-26 10:46 +0200 http://bitbucket.org/cffi/cffi/changeset/fd16324ee375/ Log: Array lengths given directly by constants diff --git a/_cffi1/parse_c_type.c b/_cffi1/parse_c_type.c --- a/_cffi1/parse_c_type.c +++ b/_cffi1/parse_c_type.c @@ -203,6 +203,8 @@ return index; } +#define MAX_SSIZE_T (((size_t)-1) >> 1) + static int parse_complete(token_t *tok); static int parse_sequel(token_t *tok, int outer) @@ -329,17 +331,44 @@ next_token(tok); if (tok->kind != TOK_CLOSE_BRACKET) { size_t length; + int gindex; - if (tok->kind != TOK_INTEGER) + switch (tok->kind) { + + case TOK_INTEGER: + errno = 0; + if (sizeof(length) > sizeof(unsigned long)) + length = strtoull(tok->p, NULL, 10); + else + length = strtoul(tok->p, NULL, 10); + if (errno == ERANGE || length > MAX_SSIZE_T) + return parse_error(tok, "number too large"); + break; + + case TOK_IDENTIFIER: + gindex = search_in_globals(tok->info->ctx, tok->p, tok->size); + if (gindex >= 0) { + const struct _cffi_global_s *g; + g = &tok->info->ctx->globals[gindex]; + if (_CFFI_GETOP(g->type_op) == _CFFI_OP_CONSTANT_INT || + _CFFI_GETOP(g->type_op) == _CFFI_OP_ENUM) { + unsigned long long value; + int neg = ((int(*)(unsigned long long*))g->address) + (&value); + if (!neg && value > MAX_SSIZE_T) + return parse_error(tok, + "integer constant too large"); + if (!neg || value == 0) { + length = (size_t)value; + break; + } + } + } + /* fall-through to the default case */ + default: return parse_error(tok, "expected a positive integer constant"); + } - errno = 0; - if (sizeof(length) > sizeof(unsigned long)) - length = strtoull(tok->p, NULL, 10); - else - length = strtoul(tok->p, NULL, 10); - if (errno == ERANGE || ((ssize_t)length) < 0) - return parse_error(tok, "number too large"); next_token(tok); write_ds(tok, _CFFI_OP(_CFFI_OP_ARRAY, 0)); diff --git a/_cffi1/realize_c_type.c b/_cffi1/realize_c_type.c --- a/_cffi1/realize_c_type.c +++ b/_cffi1/realize_c_type.c @@ -219,6 +219,8 @@ { PyObject *x; unsigned long long value; + /* note: we cast g->address to this function type; we do the same + in parse_c_type:parse_sequel() too */ int neg = ((int(*)(unsigned long long*))g->address)(&value); if (!neg) { if (value <= (unsigned long long)LONG_MAX) diff --git a/_cffi1/test_parse_c_type.py b/_cffi1/test_parse_c_type.py --- a/_cffi1/test_parse_c_type.py +++ b/_cffi1/test_parse_c_type.py @@ -1,7 +1,6 @@ -import re -import os -import py +import sys, re, os, py import cffi +from . import cffi_opcode r_macro = re.compile(r"#define \w+[(][^\n]*|#include [^\n]*") r_define = re.compile(r"(#define \w+) [^\n]*") @@ -29,6 +28,9 @@ identifier_names = ["id", "id0", "id05", "id05b", "tail"] assert identifier_names == sorted(identifier_names) +global_names = ["FIVE", "NEG", "ZERO"] +assert global_names == sorted(global_names) + ctx = ffi.new("struct _cffi_type_context_s *") c_struct_names = [ffi.new("char[]", _n) for _n in struct_names] ctx_structs = ffi.new("struct _cffi_struct_union_s[]", len(struct_names)) @@ -53,6 +55,32 @@ ctx.typenames = ctx_identifiers ctx.num_typenames = len(identifier_names) + at ffi.callback("int(unsigned long long *)") +def fetch_constant_five(p): + p[0] = 5 + return 0 + at ffi.callback("int(unsigned long long *)") +def fetch_constant_zero(p): + p[0] = 0 + return 1 + at ffi.callback("int(unsigned long long *)") +def fetch_constant_neg(p): + p[0] = 123321 + return 1 + +ctx_globals = ffi.new("struct _cffi_global_s[]", len(global_names)) +c_glob_names = [ffi.new("char[]", _n) for _n in global_names] +for _i, _fn in enumerate([fetch_constant_five, + fetch_constant_neg, + fetch_constant_zero]): + ctx_globals[_i].name = c_glob_names[_i] + ctx_globals[_i].address = _fn + ctx_globals[_i].type_op = ffi.cast("_cffi_opcode_t", + cffi_opcode.OP_CONSTANT_INT if _i != 1 + else cffi_opcode.OP_ENUM) +ctx.globals = ctx_globals +ctx.num_globals = len(global_names) + def parse(input): out = ffi.new("_cffi_opcode_t[]", 100) @@ -243,6 +271,12 @@ parse_error("int a(*)", "identifier expected", 6) parse_error("int[123456789012345678901234567890]", "number too large", 4) +def test_number_too_large(): + num_max = sys.maxsize + assert parse("char[%d]" % num_max) == [Prim(lib._CFFI_PRIM_CHAR), + '->', Array(0), num_max] + parse_error("char[%d]" % (num_max + 1), "number too large", 5) + def test_complexity_limit(): parse_error("int" + "[]" * 2500, "internal type complexity limit reached", 202) @@ -270,9 +304,15 @@ '->', Pointer(0)] def test_cffi_opcode_sync(): - import cffi_opcode, cffi.model + import cffi.model for name in dir(lib): if name.startswith('_CFFI_'): assert getattr(cffi_opcode, name[6:]) == getattr(lib, name) assert sorted(cffi_opcode.PRIMITIVE_TO_INDEX.keys()) == ( sorted(cffi.model.PrimitiveType.ALL_PRIMITIVE_TYPES.keys())) + +def test_array_length_from_constant(): + parse_error("int[UNKNOWN]", "expected a positive integer constant", 4) + assert parse("int[FIVE]") == [Prim(lib._CFFI_PRIM_INT), '->', Array(0), 5] + assert parse("int[ZERO]") == [Prim(lib._CFFI_PRIM_INT), '->', Array(0), 0] + parse_error("int[NEG]", "expected a positive integer constant", 4) From noreply at buildbot.pypy.org Sun Apr 26 10:59:40 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 26 Apr 2015 10:59:40 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: fix test Message-ID: <20150426085940.94FA81C0845@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1833:0a04e4942846 Date: 2015-04-26 10:48 +0200 http://bitbucket.org/cffi/cffi/changeset/0a04e4942846/ Log: fix test diff --git a/_cffi1/test_new_ffi_1.py b/_cffi1/test_new_ffi_1.py --- a/_cffi1/test_new_ffi_1.py +++ b/_cffi1/test_new_ffi_1.py @@ -2,7 +2,6 @@ import platform, imp import sys, ctypes import cffi -from cffi import CDefError, FFIError from .udir import udir from .recompiler import recompile from .support import * @@ -1230,8 +1229,8 @@ def test_array_of_func_ptr(self): f = ffi.cast("int(*)(int)", 42) assert f != ffi.NULL - py.test.raises(CDefError, ffi.cast, "int(int)", 42) - py.test.raises(CDefError, ffi.new, "int([5])(int)") + py.test.raises(ffi.error, ffi.cast, "int(int)", 42) + py.test.raises(ffi.error, ffi.new, "int([5])(int)") a = ffi.new("int(*[5])(int)", [f]) assert ffi.getctype(ffi.typeof(a)) == "int(*[5])(int)" assert len(a) == 5 From noreply at buildbot.pypy.org Sun Apr 26 10:59:41 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 26 Apr 2015 10:59:41 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Add the extra tests from test_ffi_backend.py Message-ID: <20150426085941.A7AE61C0845@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1834:40b66d3965df Date: 2015-04-26 10:51 +0200 http://bitbucket.org/cffi/cffi/changeset/40b66d3965df/ Log: Add the extra tests from test_ffi_backend.py diff --git a/_cffi1/test_new_ffi_1.py b/_cffi1/test_new_ffi_1.py --- a/_cffi1/test_new_ffi_1.py +++ b/_cffi1/test_new_ffi_1.py @@ -64,6 +64,7 @@ union { int c, d; }; }; struct abc50 { int a, b; int c[50]; }; + struct ints_and_bitfield { int a,b,c,d,e; int x:1; }; """ DEFS_PACKED = """ struct is_packed { char a; int b; } /*here*/; @@ -1581,3 +1582,24 @@ assert s[0].a == b'X' assert s[1].b == -4892220 assert s[1].a == b'Y' + + def test_not_supported_bitfield_in_result(self): + # struct ints_and_bitfield { int a,b,c,d,e; int x:1; }; + e = py.test.raises(NotImplementedError, ffi.callback, + "struct ints_and_bitfield foo(void)", lambda: 42) + assert str(e.value) == ("struct ints_and_bitfield(*)(): " + "callback with unsupported argument or return type or with '...'") + + def test_inspecttype(self): + assert ffi.typeof("long").kind == "primitive" + assert ffi.typeof("long(*)(long, long**, ...)").cname == ( + "long(*)(long, long * *, ...)") + assert ffi.typeof("long(*)(long, long**, ...)").ellipsis is True + + def test_new_handle(self): + o = [2, 3, 4] + p = ffi.new_handle(o) + assert ffi.typeof(p) == ffi.typeof("void *") + assert ffi.from_handle(p) is o + assert ffi.from_handle(ffi.cast("char *", p)) is o + py.test.raises(RuntimeError, ffi.from_handle, ffi.NULL) From noreply at buildbot.pypy.org Sun Apr 26 10:59:42 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 26 Apr 2015 10:59:42 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Rename the types, which are not available in the cffi module Message-ID: <20150426085942.B44B41C0845@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1835:5e6d6dfcf4e7 Date: 2015-04-26 10:55 +0200 http://bitbucket.org/cffi/cffi/changeset/5e6d6dfcf4e7/ Log: Rename the types, which are not available in the cffi module diff --git a/_cffi1/cglob.c b/_cffi1/cglob.c --- a/_cffi1/cglob.c +++ b/_cffi1/cglob.c @@ -15,7 +15,7 @@ static PyTypeObject GlobSupport_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "cffi.GlobSupport", + "FFIGlobSupport", sizeof(GlobSupportObject), 0, (destructor)glob_support_dealloc, /* tp_dealloc */ diff --git a/_cffi1/ffi_obj.c b/_cffi1/ffi_obj.c --- a/_cffi1/ffi_obj.c +++ b/_cffi1/ffi_obj.c @@ -713,7 +713,7 @@ static PyTypeObject FFI_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "cffi.FFI", + "FFI", sizeof(FFIObject), 0, (destructor)ffi_dealloc, /* tp_dealloc */ diff --git a/_cffi1/lib_obj.c b/_cffi1/lib_obj.c --- a/_cffi1/lib_obj.c +++ b/_cffi1/lib_obj.c @@ -69,7 +69,7 @@ static PyObject *lib_repr(LibObject *lib) { - return PyText_FromFormat("", + return PyText_FromFormat("", PyText_AS_UTF8(lib->l_libname)); } @@ -284,7 +284,7 @@ static PyTypeObject Lib_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "cffi.Lib", + "Lib", sizeof(LibObject), 0, (destructor)lib_dealloc, /* tp_dealloc */ From noreply at buildbot.pypy.org Sun Apr 26 10:59:43 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 26 Apr 2015 10:59:43 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: List the remaining methods that are definitely missing Message-ID: <20150426085943.B1D651C0845@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1836:f59f9e25a388 Date: 2015-04-26 11:00 +0200 http://bitbucket.org/cffi/cffi/changeset/f59f9e25a388/ Log: List the remaining methods that are definitely missing diff --git a/_cffi1/ffi_obj.c b/_cffi1/ffi_obj.c --- a/_cffi1/ffi_obj.c +++ b/_cffi1/ffi_obj.c @@ -689,14 +689,23 @@ {"__set_types",(PyCFunction)ffi__set_types, METH_VARARGS}, {"addressof", (PyCFunction)ffi_addressof, METH_VARARGS, ffi_addressof_doc}, {"alignof", (PyCFunction)ffi_alignof, METH_O, ffi_alignof_doc}, +#if 0 + {"buffer", (PyCFunction)ffi_buffer, METH_VARARGS, ffi_buffer_doc}, +#endif {"callback", (PyCFunction)ffi_callback, METH_VARARGS | METH_KEYWORDS,ffi_callback_doc}, {"cast", (PyCFunction)ffi_cast, METH_VARARGS, ffi_cast_doc}, +#if 0 + {"from_buffer",(PyCFunction)ffi_from_buffer,METH_O, ffi_from_buffer_doc}, +#endif {"from_handle",(PyCFunction)ffi_from_handle,METH_O, ffi_from_handle_doc}, #if 0 {"gc", (PyCFunction)ffi_gc, METH_VARARGS}, #endif {"getctype", (PyCFunction)ffi_getctype, METH_VARARGS, ffi_getctype_doc}, +#if 0 + {"getwinerror",(PyCFunction)ffi_getwinerror,METH_VARARGS, ffi_getwinerror_doc}, +#endif {"offsetof", (PyCFunction)ffi_offsetof, METH_VARARGS, ffi_offsetof_doc}, {"new", (PyCFunction)ffi_new, METH_VARARGS, ffi_new_doc}, {"new_handle", (PyCFunction)ffi_new_handle, METH_O, ffi_new_handle_doc}, From noreply at buildbot.pypy.org Sun Apr 26 11:23:26 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 26 Apr 2015 11:23:26 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: ffi.getwinerror() (untested so far) Message-ID: <20150426092326.F02451C0359@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1837:bc882378fefd Date: 2015-04-26 11:05 +0200 http://bitbucket.org/cffi/cffi/changeset/bc882378fefd/ Log: ffi.getwinerror() (untested so far) diff --git a/_cffi1/ffi_obj.c b/_cffi1/ffi_obj.c --- a/_cffi1/ffi_obj.c +++ b/_cffi1/ffi_obj.c @@ -581,6 +581,15 @@ return res; } +#ifdef MS_WIN32 +PyDoc_STRVAR(ffi_getwinerror_doc, +"Return either the GetLastError() or the error number given by the\n" +"optional 'code' argument, as a tuple '(code, message)'."); + +#define ffi_getwinerror b_getwinerror /* ffi_getwinerror() => b_getwinerror() + from misc_win32.h */ +#endif + PyDoc_STRVAR(ffi_errno_doc, "the value of 'errno' from/to the C calls"); static PyObject *ffi_get_errno(PyObject *self, void *closure) @@ -703,7 +712,7 @@ {"gc", (PyCFunction)ffi_gc, METH_VARARGS}, #endif {"getctype", (PyCFunction)ffi_getctype, METH_VARARGS, ffi_getctype_doc}, -#if 0 +#ifdef MS_WIN32 {"getwinerror",(PyCFunction)ffi_getwinerror,METH_VARARGS, ffi_getwinerror_doc}, #endif {"offsetof", (PyCFunction)ffi_offsetof, METH_VARARGS, ffi_offsetof_doc}, From noreply at buildbot.pypy.org Sun Apr 26 11:23:28 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 26 Apr 2015 11:23:28 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: ffi.buffer() Message-ID: <20150426092328.0FCC41C0359@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1838:c1a8a13f781d Date: 2015-04-26 11:09 +0200 http://bitbucket.org/cffi/cffi/changeset/c1a8a13f781d/ Log: ffi.buffer() diff --git a/_cffi1/ffi_obj.c b/_cffi1/ffi_obj.c --- a/_cffi1/ffi_obj.c +++ b/_cffi1/ffi_obj.c @@ -302,6 +302,20 @@ #define ffi_string b_string /* ffi_string() => b_string() from _cffi_backend.c */ +PyDoc_STRVAR(ffi_buffer_doc, +"Return a read-write buffer object that references the raw C data\n" +"pointed to by the given 'cdata'. The 'cdata' must be a pointer or an\n" +"array. Can be passed to functions expecting a buffer, or directly\n" +"manipulated with:\n" +"\n" +" buf[:] get a copy of it in a regular string, or\n" +" buf[idx] as a single character\n" +" buf[:] = ...\n" +" buf[idx] = ... change the content"); + +#define ffi_buffer b_buffer /* ffi_buffer() => b_buffer() + from _cffi_backend.c */ + PyDoc_STRVAR(ffi_offsetof_doc, "Return the offset of the named field inside the given structure or\n" "array, which must be given as a C type name. You can give several\n" @@ -698,9 +712,7 @@ {"__set_types",(PyCFunction)ffi__set_types, METH_VARARGS}, {"addressof", (PyCFunction)ffi_addressof, METH_VARARGS, ffi_addressof_doc}, {"alignof", (PyCFunction)ffi_alignof, METH_O, ffi_alignof_doc}, -#if 0 {"buffer", (PyCFunction)ffi_buffer, METH_VARARGS, ffi_buffer_doc}, -#endif {"callback", (PyCFunction)ffi_callback, METH_VARARGS | METH_KEYWORDS,ffi_callback_doc}, {"cast", (PyCFunction)ffi_cast, METH_VARARGS, ffi_cast_doc}, From noreply at buildbot.pypy.org Sun Apr 26 11:23:29 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 26 Apr 2015 11:23:29 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: more tests from test_ffi_backend Message-ID: <20150426092329.1E7141C0359@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1839:e80b46e5fa79 Date: 2015-04-26 11:11 +0200 http://bitbucket.org/cffi/cffi/changeset/e80b46e5fa79/ Log: more tests from test_ffi_backend diff --git a/_cffi1/test_new_ffi_1.py b/_cffi1/test_new_ffi_1.py --- a/_cffi1/test_new_ffi_1.py +++ b/_cffi1/test_new_ffi_1.py @@ -54,6 +54,7 @@ typedef struct { int a; } unnamed_foo_t, *unnamed_foo_p; struct nonpacked { char a; int b; }; struct array0 { int len; short data[0]; }; + struct array_no_length { int x; int a[]; }; struct nested_anon { struct { int a, b; }; @@ -1603,3 +1604,20 @@ assert ffi.from_handle(p) is o assert ffi.from_handle(ffi.cast("char *", p)) is o py.test.raises(RuntimeError, ffi.from_handle, ffi.NULL) + + def test_struct_array_no_length(self): + # struct array_no_length { int x; int a[]; }; + p = ffi.new("struct array_no_length *", [100, [200, 300, 400]]) + assert p.x == 100 + assert ffi.typeof(p.a) is ffi.typeof("int *") # no length available + assert p.a[0] == 200 + assert p.a[1] == 300 + assert p.a[2] == 400 + + def test_from_buffer(self): + import array + a = array.array('H', [10000, 20000, 30000]) + c = ffi.from_buffer(a) + assert ffi.typeof(c) is ffi.typeof("char[]") + ffi.cast("unsigned short *", c)[1] += 500 + assert list(a) == [10000, 20500, 30000] From noreply at buildbot.pypy.org Sun Apr 26 11:23:30 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 26 Apr 2015 11:23:30 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: ffi.from_buffer() Message-ID: <20150426092330.2D2D11C0359@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1840:923f1d3495c9 Date: 2015-04-26 11:23 +0200 http://bitbucket.org/cffi/cffi/changeset/923f1d3495c9/ Log: ffi.from_buffer() diff --git a/_cffi1/ffi_obj.c b/_cffi1/ffi_obj.c --- a/_cffi1/ffi_obj.c +++ b/_cffi1/ffi_obj.c @@ -533,6 +533,19 @@ return x; } +PyDoc_STRVAR(ffi_from_buffer_doc, +"Return a that points to the data of the given Python\n" +"object, which must support the buffer interface. Note that this is\n" +"not meant to be used on the built-in types str, unicode, or bytearray\n" +"(you can build 'char[]' arrays explicitly) but only on objects\n" +"containing large quantities of raw data in some other format, like\n" +"'array.array' or numpy arrays."); + +static PyObject *ffi_from_buffer(PyObject *self, PyObject *arg) +{ + return direct_from_buffer(g_ct_chararray, arg); +} + #if 0 static PyObject *ffi_gc(ZefFFIObject *self, PyObject *args) { @@ -716,9 +729,7 @@ {"callback", (PyCFunction)ffi_callback, METH_VARARGS | METH_KEYWORDS,ffi_callback_doc}, {"cast", (PyCFunction)ffi_cast, METH_VARARGS, ffi_cast_doc}, -#if 0 {"from_buffer",(PyCFunction)ffi_from_buffer,METH_O, ffi_from_buffer_doc}, -#endif {"from_handle",(PyCFunction)ffi_from_handle,METH_O, ffi_from_handle_doc}, #if 0 {"gc", (PyCFunction)ffi_gc, METH_VARARGS}, diff --git a/_cffi1/realize_c_type.c b/_cffi1/realize_c_type.c --- a/_cffi1/realize_c_type.c +++ b/_cffi1/realize_c_type.c @@ -8,7 +8,7 @@ static PyObject *all_primitives[_CFFI__NUM_PRIM]; static PyObject *global_types_dict; -static CTypeDescrObject *g_ct_voidp; +static CTypeDescrObject *g_ct_voidp, *g_ct_chararray; static PyObject *build_primitive_type(int num); /* forward */ @@ -16,33 +16,49 @@ (all_primitives[num] != NULL ? all_primitives[num] \ : build_primitive_type(num)) +static int _add_to_global_types_dict(PyObject *ct) +{ + if (ct == NULL) + return -1; + return PyDict_SetItemString(global_types_dict, + ((CTypeDescrObject *)ct)->ct_name, ct); +} + static int init_global_types_dict(PyObject *ffi_type_dict) { int err; - PyObject *ct, *ct2, *pnull; + PyObject *ct_void, *ct_char, *ct2, *pnull; + /* XXX some leaks in case these functions fail, but well, + MemoryErrors during importing an extension module are kind + of bad anyway */ global_types_dict = PyDict_New(); if (global_types_dict == NULL) return -1; - ct = get_primitive_type(_CFFI_PRIM_VOID); // 'void' - if (ct == NULL) + ct_void = get_primitive_type(_CFFI_PRIM_VOID); // 'void' + if (_add_to_global_types_dict(ct_void) < 0) return -1; - if (PyDict_SetItemString(global_types_dict, - ((CTypeDescrObject *)ct)->ct_name, ct) < 0) { + + ct2 = new_pointer_type((CTypeDescrObject *)ct_void); // 'void *' + if (_add_to_global_types_dict(ct2) < 0) return -1; - } - ct2 = new_pointer_type((CTypeDescrObject *)ct); // 'void *' - if (ct2 == NULL) - return -1; - if (PyDict_SetItemString(global_types_dict, - ((CTypeDescrObject *)ct2)->ct_name, ct2) < 0) { - Py_DECREF(ct2); - return -1; - } g_ct_voidp = (CTypeDescrObject *)ct2; - pnull = new_simple_cdata(NULL, (CTypeDescrObject *)ct2); + ct_char = get_primitive_type(_CFFI_PRIM_CHAR); // 'char' + if (_add_to_global_types_dict(ct_char) < 0) + return -1; + + ct2 = new_pointer_type((CTypeDescrObject *)ct_char); // 'char *' + if (_add_to_global_types_dict(ct2) < 0) + return -1; + + ct2 = new_array_type((CTypeDescrObject *)ct2, -1); // 'char[]' + if (_add_to_global_types_dict(ct2) < 0) + return -1; + g_ct_chararray = (CTypeDescrObject *)ct2; + + pnull = new_simple_cdata(NULL, g_ct_voidp); if (pnull == NULL) return -1; err = PyDict_SetItemString(ffi_type_dict, "NULL", pnull); diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -5359,21 +5359,11 @@ return 0; } -static PyObject *b_from_buffer(PyObject *self, PyObject *args) -{ - CTypeDescrObject *ct; +static PyObject *direct_from_buffer(CTypeDescrObject *ct, PyObject *x) +{ CDataObject *cd; - PyObject *x; Py_buffer *view; - if (!PyArg_ParseTuple(args, "O!O", &CTypeDescr_Type, &ct, &x)) - return NULL; - - if (!(ct->ct_flags & CT_IS_UNSIZED_CHAR_A)) { - PyErr_Format(PyExc_TypeError, "needs 'char[]', got '%s'", ct->ct_name); - return NULL; - } - if (invalid_input_buffer_type(x)) { PyErr_SetString(PyExc_TypeError, "from_buffer() cannot return the address of the " @@ -5407,6 +5397,21 @@ return NULL; } +static PyObject *b_from_buffer(PyObject *self, PyObject *args) +{ + CTypeDescrObject *ct; + PyObject *x; + + if (!PyArg_ParseTuple(args, "O!O", &CTypeDescr_Type, &ct, &x)) + return NULL; + + if (!(ct->ct_flags & CT_IS_UNSIZED_CHAR_A)) { + PyErr_Format(PyExc_TypeError, "needs 'char[]', got '%s'", ct->ct_name); + return NULL; + } + return direct_from_buffer(ct, x); +} + static PyObject *b__get_types(PyObject *self, PyObject *noarg) { return PyTuple_Pack(2, (PyObject *)&CData_Type, From noreply at buildbot.pypy.org Sun Apr 26 11:29:08 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 26 Apr 2015 11:29:08 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: ffi.gc(), copied straight from zeffir Message-ID: <20150426092908.B33B61C0359@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1841:e0ca28cfacbd Date: 2015-04-26 11:29 +0200 http://bitbucket.org/cffi/cffi/changeset/e0ca28cfacbd/ Log: ffi.gc(), copied straight from zeffir diff --git a/_cffi1/cffi1_module.c b/_cffi1/cffi1_module.c --- a/_cffi1/cffi1_module.c +++ b/_cffi1/cffi1_module.c @@ -10,6 +10,7 @@ #include "ffi_obj.c" #include "cglob.c" +#include "cgc.c" #include "lib_obj.c" diff --git a/_cffi1/cgc.c b/_cffi1/cgc.c new file mode 100644 --- /dev/null +++ b/_cffi1/cgc.c @@ -0,0 +1,80 @@ + +/* translated to C from cffi/gc_weakref.py */ + + +static PyObject *const_name_pop; + +static PyObject *gc_wref_remove(PyObject *ffi_wref_data, PyObject *arg) +{ + PyObject *destructor, *cdata, *x; + PyObject *res = PyObject_CallMethodObjArgs(ffi_wref_data, + const_name_pop, arg, NULL); + if (res == NULL) + return NULL; + + assert(PyTuple_Check(res)); + destructor = PyTuple_GET_ITEM(res, 0); + cdata = PyTuple_GET_ITEM(res, 1); + x = PyObject_CallFunctionObjArgs(destructor, cdata, NULL); + Py_DECREF(res); + if (x == NULL) + return NULL; + Py_DECREF(x); + + Py_INCREF(Py_None); + return Py_None; +} + +static PyMethodDef remove_callback = { + "gc_wref_remove", (PyCFunction)gc_wref_remove, METH_O +}; + +static PyObject *gc_weakrefs_build(FFIObject *ffi, CDataObject *cd, + PyObject *destructor) +{ + PyObject *new_cdata, *ref = NULL, *tup = NULL; + + if (ffi->gc_wrefs == NULL) { + /* initialize */ + PyObject *data; + + if (const_name_pop == NULL) { + const_name_pop = PyString_InternFromString("pop"); + if (const_name_pop == NULL) + return NULL; + } + data = PyDict_New(); + if (data == NULL) + return NULL; + ffi->gc_wrefs = PyCFunction_New(&remove_callback, data); + Py_DECREF(data); + if (ffi->gc_wrefs == NULL) + return NULL; + } + + new_cdata = do_cast(cd->c_type, (PyObject *)cd); + if (new_cdata == NULL) + goto error; + + ref = PyWeakref_NewRef(new_cdata, ffi->gc_wrefs); + if (ref == NULL) + goto error; + + tup = PyTuple_Pack(2, destructor, cd); + if (tup == NULL) + goto error; + + /* the 'self' of the function 'gc_wrefs' is actually the data dict */ + if (PyDict_SetItem(PyCFunction_GET_SELF(ffi->gc_wrefs), ref, tup) < 0) + goto error; + + Py_DECREF(tup); + Py_DECREF(ref); + return new_cdata; + + error: + Py_XDECREF(new_cdata); + Py_XDECREF(ref); + Py_XDECREF(tup); + return NULL; +} diff --git a/_cffi1/ffi_obj.c b/_cffi1/ffi_obj.c --- a/_cffi1/ffi_obj.c +++ b/_cffi1/ffi_obj.c @@ -546,8 +546,15 @@ return direct_from_buffer(g_ct_chararray, arg); } -#if 0 -static PyObject *ffi_gc(ZefFFIObject *self, PyObject *args) +PyDoc_STRVAR(ffi_gc_doc, +"Return a new cdata object that points to the same data.\n" +"Later, when this new cdata object is garbage-collected,\n" +"'destructor(old_cdata_object)' will be called."); + +static PyObject *gc_weakrefs_build(FFIObject *ffi, CDataObject *cd, + PyObject *destructor); /* forward */ + +static PyObject *ffi_gc(FFIObject *self, PyObject *args) { CDataObject *cd; PyObject *destructor; @@ -557,7 +564,6 @@ return gc_weakrefs_build(self, cd, destructor); } -#endif PyDoc_STRVAR(ffi_callback_doc, "Return a callback object or a decorator making such a callback object.\n" @@ -731,9 +737,7 @@ {"cast", (PyCFunction)ffi_cast, METH_VARARGS, ffi_cast_doc}, {"from_buffer",(PyCFunction)ffi_from_buffer,METH_O, ffi_from_buffer_doc}, {"from_handle",(PyCFunction)ffi_from_handle,METH_O, ffi_from_handle_doc}, -#if 0 - {"gc", (PyCFunction)ffi_gc, METH_VARARGS}, -#endif + {"gc", (PyCFunction)ffi_gc, METH_VARARGS, ffi_gc_doc}, {"getctype", (PyCFunction)ffi_getctype, METH_VARARGS, ffi_getctype_doc}, #ifdef MS_WIN32 {"getwinerror",(PyCFunction)ffi_getwinerror,METH_VARARGS, ffi_getwinerror_doc}, From noreply at buildbot.pypy.org Sun Apr 26 11:37:07 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 26 Apr 2015 11:37:07 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Move and fix the test Message-ID: <20150426093707.3077C1C0359@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1842:3919dd27269f Date: 2015-04-26 11:37 +0200 http://bitbucket.org/cffi/cffi/changeset/3919dd27269f/ Log: Move and fix the test diff --git a/_cffi1/test_new_ffi_1.py b/_cffi1/test_new_ffi_1.py --- a/_cffi1/test_new_ffi_1.py +++ b/_cffi1/test_new_ffi_1.py @@ -1278,26 +1278,6 @@ sz = ffi.sizeof("long") assert cb((1 << (sz*8-1)) - 1, -10) == 42 - def test_unique_types(self): - xxxx - ffi1 = FFI(backend=self.Backend()) - ffi2 = FFI(backend=self.Backend()) - assert ffi1.typeof("char") is ffi2.typeof("char ") - assert ffi1.typeof("long") is ffi2.typeof("signed long int") - assert ffi1.typeof("double *") is ffi2.typeof("double*") - assert ffi1.typeof("int ***") is ffi2.typeof(" int * * *") - assert ffi1.typeof("int[]") is ffi2.typeof("signed int[]") - assert ffi1.typeof("signed int*[17]") is ffi2.typeof("int *[17]") - assert ffi1.typeof("void") is ffi2.typeof("void") - assert ffi1.typeof("int(*)(int,int)") is ffi2.typeof("int(*)(int,int)") - # - # these depend on user-defined data, so should not be shared - assert ffi1.typeof("struct foo") is not ffi2.typeof("struct foo") - assert ffi1.typeof("union foo *") is not ffi2.typeof("union foo*") - assert ffi1.typeof("enum foo") is not ffi2.typeof("enum foo") - # sanity check: twice 'ffi1' - assert ffi1.typeof("struct foo*") is ffi1.typeof("struct foo *") - def test_anonymous_enum(self): # typedef enum { Value0 = 0 } e_t, *pe_t; assert ffi.getctype("e_t*") == 'e_t *' diff --git a/_cffi1/test_recompiler.py b/_cffi1/test_recompiler.py --- a/_cffi1/test_recompiler.py +++ b/_cffi1/test_recompiler.py @@ -374,3 +374,29 @@ "typedef enum { AA=%d } e1;" % sys.maxint) assert lib.AA == sys.maxint assert ffi.sizeof("e1") == ffi.sizeof("long") + +def test_unique_types(): + CDEF = "struct foo_s; union foo_u; enum foo_e { AA };" + ffi1 = FFI(); ffi1.cdef(CDEF); verify(ffi1, "test_unique_types_1", CDEF) + ffi2 = FFI(); ffi2.cdef(CDEF); verify(ffi2, "test_unique_types_2", CDEF) + # + assert ffi1.typeof("char") is ffi2.typeof("char ") + assert ffi1.typeof("long") is ffi2.typeof("signed long int") + assert ffi1.typeof("double *") is ffi2.typeof("double*") + assert ffi1.typeof("int ***") is ffi2.typeof(" int * * *") + assert ffi1.typeof("int[]") is ffi2.typeof("signed int[]") + assert ffi1.typeof("signed int*[17]") is ffi2.typeof("int *[17]") + assert ffi1.typeof("void") is ffi2.typeof("void") + assert ffi1.typeof("int(*)(int,int)") is ffi2.typeof("int(*)(int,int)") + # + # these depend on user-defined data, so should not be shared + for name in ["struct foo_s", + "union foo_u *", + "enum foo_e", + "struct foo_s *(*)()", + "void(*)(struct foo_s *)", + "struct foo_s *(*[5])[8]", + ]: + assert ffi1.typeof(name) is not ffi2.typeof(name) + # sanity check: twice 'ffi1' + assert ffi1.typeof("struct foo_s*") is ffi1.typeof("struct foo_s *") From noreply at buildbot.pypy.org Sun Apr 26 11:51:46 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 26 Apr 2015 11:51:46 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: small refactoring ending up with support for SF_PACKED structures Message-ID: <20150426095146.4685F1C0359@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1843:fd13bd62a49a Date: 2015-04-26 11:52 +0200 http://bitbucket.org/cffi/cffi/changeset/fd13bd62a49a/ Log: small refactoring ending up with support for SF_PACKED structures diff --git a/_cffi1/cffi_opcode.py b/_cffi1/cffi_opcode.py --- a/_cffi1/cffi_opcode.py +++ b/_cffi1/cffi_opcode.py @@ -97,6 +97,10 @@ 'ssize_t': PRIM_SSIZE, } +F_UNION = 0x01 +F_CHECK_FIELDS = 0x02 +F_PACKED = 0x04 + CLASS_NAME = {} for _name, _value in globals().items(): if _name.startswith('OP_') and isinstance(_value, int): diff --git a/_cffi1/ffi_obj.c b/_cffi1/ffi_obj.c --- a/_cffi1/ffi_obj.c +++ b/_cffi1/ffi_obj.c @@ -692,7 +692,8 @@ if (!CTypeDescr_Check(x)) goto bad_usage; types[i] = x; - struct_unions[i].flags = ((CTypeDescrObject *)x)->ct_flags & CT_UNION; + struct_unions[i].flags = ((CTypeDescrObject *)x)->ct_flags & CT_UNION ? + _CFFI_F_UNION : 0; struct_unions[i].size = (size_t)-2; struct_unions[i].alignment = -2; } diff --git a/_cffi1/parse_c_type.c b/_cffi1/parse_c_type.c --- a/_cffi1/parse_c_type.c +++ b/_cffi1/parse_c_type.c @@ -628,7 +628,7 @@ int n = search_in_struct_unions(tok->info->ctx, tok->p, tok->size); if (n < 0) return parse_error(tok, "undefined struct/union name"); - if (((tok->info->ctx->struct_unions[n].flags & CT_UNION) != 0) + if (((tok->info->ctx->struct_unions[n].flags & _CFFI_F_UNION) != 0) ^ (kind == TOK_UNION)) return parse_error(tok, "wrong kind of tag: struct vs union"); diff --git a/_cffi1/parse_c_type.h b/_cffi1/parse_c_type.h --- a/_cffi1/parse_c_type.h +++ b/_cffi1/parse_c_type.h @@ -69,18 +69,16 @@ struct _cffi_struct_union_s { const char *name; int type_index; // -> _cffi_types, on a OP_STRUCT_UNION - int flags; // CT_UNION? CT_CUSTOM_FIELD_POS? + int flags; // _CFFI_F_* flags below size_t size; int alignment; int first_field_index; // -> _cffi_fields array int num_fields; }; -#ifdef _CFFI_INTERNAL -#define CT_UNION 128 -#define CT_CUSTOM_FIELD_POS 32768 -/* ^^^ if not CUSTOM_FIELD_POS, complain if fields are not in the - "standard layout" and/or if some are missing */ -#endif +#define _CFFI_F_UNION 0x01 // is a union, not a struct +#define _CFFI_F_CHECK_FIELDS 0x02 // complain if fields are not in the + // "standard layout" or if some are missing +#define _CFFI_F_PACKED 0x04 // for CHECK_FIELDS, assume a packed struct struct _cffi_field_s { const char *name; diff --git a/_cffi1/realize_c_type.c b/_cffi1/realize_c_type.c --- a/_cffi1/realize_c_type.c +++ b/_cffi1/realize_c_type.c @@ -404,10 +404,10 @@ Py_INCREF(x); } else { - int flags = (s->flags & CT_UNION) ? CT_UNION : CT_STRUCT; + int flags = (s->flags & _CFFI_F_UNION) ? CT_UNION : CT_STRUCT; char *name = alloca(8 + strlen(s->name)); _realize_name(name, - (s->flags & CT_UNION) ? "union " : "struct ", + (s->flags & _CFFI_F_UNION) ? "union " : "struct ", s->name); x = new_struct_or_union_type(name, flags); @@ -683,7 +683,11 @@ PyList_SET_ITEM(fields, i, f); } - int sflags = (s->flags & CT_CUSTOM_FIELD_POS) ? 0 : SF_STD_FIELD_POS; + int sflags = 0; + if (s->flags & _CFFI_F_CHECK_FIELDS) + sflags |= SF_STD_FIELD_POS; + if (s->flags & _CFFI_F_PACKED) + sflags |= SF_PACKED; PyObject *args = Py_BuildValue("(OOOnni)", ct, fields, Py_None, @@ -696,7 +700,6 @@ ct->ct_extra = NULL; ct->ct_flags |= CT_IS_OPAQUE; - ct->ct_flags &= ~CT_CUSTOM_FIELD_POS; PyObject *res = b_complete_struct_or_union(NULL, args); ct->ct_flags &= ~CT_IS_OPAQUE; Py_DECREF(args); diff --git a/_cffi1/recompiler.py b/_cffi1/recompiler.py --- a/_cffi1/recompiler.py +++ b/_cffi1/recompiler.py @@ -456,11 +456,16 @@ def _struct_ctx(self, tp, cname, approxname): type_index = self._typesdict[tp] - flags = 0 + flags = [] + if isinstance(tp, model.UnionType): + flags.append("_CFFI_F_UNION") if tp.partial or tp.has_anonymous_struct_fields(): - flags |= 32768 # CT_CUSTOM_FIELD_POS - if isinstance(tp, model.UnionType): - flags |= 128 # CT_UNION + pass # the field layout is obtained silently from the C compiler + else: + flags.append("_CFFI_F_CHECK_FIELDS") + if tp.packed: + flags.append("_CFFI_F_PACKED") + flags = '|'.join(flags) or '0' if tp.fldtypes is not None: c_field = [approxname] enumfields = list(tp.enumfields()) @@ -502,7 +507,7 @@ else: size_align = ' (size_t)-1, -1, -1, 0 /* opaque */ },' self._lsts["struct_union"].append( - ' { "%s", %d, 0x%x,' % (tp.name, type_index, flags) + size_align) + ' { "%s", %d, %s,' % (tp.name, type_index, flags) + size_align) self._seen_struct_unions.add(tp) def _add_missing_struct_unions(self): diff --git a/_cffi1/test_new_ffi_1.py b/_cffi1/test_new_ffi_1.py --- a/_cffi1/test_new_ffi_1.py +++ b/_cffi1/test_new_ffi_1.py @@ -75,9 +75,9 @@ ffi1.cdef(DEFS) ffi1.cdef(DEFS_PACKED, packed=True) - outputfilename = recompile(ffi1, "test_old_ffi1", CCODE, + outputfilename = recompile(ffi1, "test_new_ffi_1", CCODE, tmpdir=str(udir)) - module = imp.load_dynamic("test_old_ffi1", outputfilename) + module = imp.load_dynamic("test_new_ffi_1", outputfilename) ffi = module.ffi diff --git a/_cffi1/test_parse_c_type.py b/_cffi1/test_parse_c_type.py --- a/_cffi1/test_parse_c_type.py +++ b/_cffi1/test_parse_c_type.py @@ -36,7 +36,7 @@ ctx_structs = ffi.new("struct _cffi_struct_union_s[]", len(struct_names)) for _i in range(len(struct_names)): ctx_structs[_i].name = c_struct_names[_i] -ctx_structs[3].flags = lib.CT_UNION +ctx_structs[3].flags = lib._CFFI_F_UNION ctx.struct_unions = ctx_structs ctx.num_struct_unions = len(struct_names) From noreply at buildbot.pypy.org Sun Apr 26 11:56:12 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 26 Apr 2015 11:56:12 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: test skips Message-ID: <20150426095612.7C7C31C0359@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1844:71537992434b Date: 2015-04-26 11:56 +0200 http://bitbucket.org/cffi/cffi/changeset/71537992434b/ Log: test skips diff --git a/_cffi1/test_verify1.py b/_cffi1/test_verify1.py --- a/_cffi1/test_verify1.py +++ b/_cffi1/test_verify1.py @@ -2079,6 +2079,7 @@ assert ffi.getwinerror()[0] == n def test_verify_dlopen_flags(): + py.test.xfail("dlopen flags") # Careful with RTLD_GLOBAL. If by chance the FFI is not deleted # promptly, like on PyPy, then other tests may see the same # exported symbols as well. So we must not export a simple name @@ -2157,13 +2158,6 @@ assert ord(outbuf[n]) == 0 assert ord(outbuf[0]) < 128 # should be a letter, or '\' -def test_use_local_dir(): - ffi = FFI() - lib = ffi.verify("", modulename="test_use_local_dir") - this_dir = os.path.dirname(__file__) - pycache_files = os.listdir(os.path.join(this_dir, '__pycache__')) - assert any('test_use_local_dir' in s for s in pycache_files) - def test_define_known_value(): ffi = FFI() ffi.cdef("#define FOO 0x123") @@ -2173,5 +2167,5 @@ def test_define_wrong_value(): ffi = FFI() ffi.cdef("#define FOO 123") - e = py.test.raises(VerificationError, ffi.verify, "#define FOO 124") - assert str(e.value).endswith("FOO has the real value 124, not 123") + lib = ffi.verify("#define FOO 124") # used to complain + assert lib.FOO == 124 From noreply at buildbot.pypy.org Sun Apr 26 13:49:09 2015 From: noreply at buildbot.pypy.org (mjacob) Date: Sun, 26 Apr 2015 13:49:09 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: Change some whitespace to make a new revision to test something on the new build slave. Message-ID: <20150426114909.44C651C116B@cobra.cs.uni-duesseldorf.de> Author: Manuel Jacob Branch: py3.3 Changeset: r76938:975e8a2af096 Date: 2015-04-26 13:48 +0200 http://bitbucket.org/pypy/pypy/changeset/975e8a2af096/ Log: Change some whitespace to make a new revision to test something on the new build slave. diff --git a/rpython/translator/c/src/signals.c b/rpython/translator/c/src/signals.c --- a/rpython/translator/c/src/signals.c +++ b/rpython/translator/c/src/signals.c @@ -95,12 +95,12 @@ #endif if (wakeup_with_nul_byte) { - res = write(wakeup_fd, "\0", 1); + res = write(wakeup_fd, "\0", 1); } else { unsigned char byte = (unsigned char)signum; - res = write(wakeup_fd, &byte, 1); + res = write(wakeup_fd, &byte, 1); } - + /* the return value is ignored here */ } } From noreply at buildbot.pypy.org Sun Apr 26 14:11:50 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 26 Apr 2015 14:11:50 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Hex/octal numbers in array lengths Message-ID: <20150426121150.4461A1C03F7@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1845:4d94d2c57548 Date: 2015-04-26 14:11 +0200 http://bitbucket.org/cffi/cffi/changeset/4d94d2c57548/ Log: Hex/octal numbers in array lengths diff --git a/_cffi1/parse_c_type.c b/_cffi1/parse_c_type.c --- a/_cffi1/parse_c_type.c +++ b/_cffi1/parse_c_type.c @@ -69,6 +69,13 @@ return ('0' <= x && x <= '9'); } +static int is_hex_digit(char x) +{ + return (('0' <= x && x <= '9') || + ('A' <= x && x <= 'F') || + ('a' <= x && x <= 'f')); +} + static int is_ident_next(char x) { return (is_ident_first(x) || is_digit(x)); @@ -113,7 +120,9 @@ tok->kind = TOK_INTEGER; tok->p = p; tok->size = 1; - while (is_digit(p[tok->size])) + if (p[1] == 'x' || p[1] == 'X') + tok->size = 2; + while (is_hex_digit(p[tok->size])) tok->size++; return; } @@ -332,15 +341,18 @@ if (tok->kind != TOK_CLOSE_BRACKET) { size_t length; int gindex; + char *endptr; switch (tok->kind) { case TOK_INTEGER: errno = 0; if (sizeof(length) > sizeof(unsigned long)) - length = strtoull(tok->p, NULL, 10); + length = strtoull(tok->p, &endptr, 0); else - length = strtoul(tok->p, NULL, 10); + length = strtoul(tok->p, &endptr, 0); + if (endptr != tok->p + tok->size) + return parse_error(tok, "invalid number"); if (errno == ERANGE || length > MAX_SSIZE_T) return parse_error(tok, "number too large"); break; diff --git a/_cffi1/test_parse_c_type.py b/_cffi1/test_parse_c_type.py --- a/_cffi1/test_parse_c_type.py +++ b/_cffi1/test_parse_c_type.py @@ -316,3 +316,21 @@ assert parse("int[FIVE]") == [Prim(lib._CFFI_PRIM_INT), '->', Array(0), 5] assert parse("int[ZERO]") == [Prim(lib._CFFI_PRIM_INT), '->', Array(0), 0] parse_error("int[NEG]", "expected a positive integer constant", 4) + +def test_various_constant_exprs(): + def array(n): + return [Prim(lib._CFFI_PRIM_CHAR), '->', Array(0), n] + assert parse("char[21]") == array(21) + assert parse("char[0x10]") == array(16) + assert parse("char[0X21]") == array(33) + assert parse("char[0Xb]") == array(11) + assert parse("char[0x1C]") == array(0x1C) + assert parse("char[0xc6]") == array(0xC6) + assert parse("char[010]") == array(8) + assert parse("char[021]") == array(17) + parse_error("char[08]", "invalid number", 5) + parse_error("char[1C]", "invalid number", 5) + parse_error("char[0C]", "invalid number", 5) + # not supported (really obscure): + # "char[+5]" + # "char['A']" From noreply at buildbot.pypy.org Sun Apr 26 15:19:30 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 26 Apr 2015 15:19:30 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: a leak Message-ID: <20150426131930.EA2DA1C0EC0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1846:d708ab8e01c6 Date: 2015-04-26 15:20 +0200 http://bitbucket.org/cffi/cffi/changeset/d708ab8e01c6/ Log: a leak diff --git a/_cffi1/realize_c_type.c b/_cffi1/realize_c_type.c --- a/_cffi1/realize_c_type.c +++ b/_cffi1/realize_c_type.c @@ -657,6 +657,7 @@ break; default: + Py_DECREF(fields); PyErr_Format(PyExc_NotImplementedError, "field op=%d", (int)_CFFI_GETOP(op)); return -1; @@ -671,8 +672,10 @@ else if (detect_custom_layout(ct, SF_STD_FIELD_POS, ctf->ct_size, fld->field_size, "wrong size for field '", - fld->name, "'") < 0) + fld->name, "'") < 0) { + Py_DECREF(fields); return -1; + } f = Py_BuildValue("(sOin)", fld->name, ctf, fbitsize, (Py_ssize_t)fld->field_offset); From noreply at buildbot.pypy.org Sun Apr 26 15:59:39 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 26 Apr 2015 15:59:39 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: xfail the last two failures Message-ID: <20150426135939.957631C116B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1848:798a6d0fee0d Date: 2015-04-26 16:00 +0200 http://bitbucket.org/cffi/cffi/changeset/798a6d0fee0d/ Log: xfail the last two failures diff --git a/_cffi1/test_verify1.py b/_cffi1/test_verify1.py --- a/_cffi1/test_verify1.py +++ b/_cffi1/test_verify1.py @@ -1645,6 +1645,8 @@ assert repr(lib.fooarray).startswith(" Author: Armin Rigo Branch: cffi-1.0 Changeset: r1847:1e0789c75797 Date: 2015-04-26 15:56 +0200 http://bitbucket.org/cffi/cffi/changeset/1e0789c75797/ Log: FILE in the recompiler diff --git a/_cffi1/realize_c_type.c b/_cffi1/realize_c_type.c --- a/_cffi1/realize_c_type.c +++ b/_cffi1/realize_c_type.c @@ -409,6 +409,9 @@ _realize_name(name, (s->flags & _CFFI_F_UNION) ? "union " : "struct ", s->name); + if (strcmp(name, "struct _IO_FILE") == 0) + flags |= CT_IS_FILE; + x = new_struct_or_union_type(name, flags); CTypeDescrObject *ct = NULL; diff --git a/_cffi1/recompiler.py b/_cffi1/recompiler.py --- a/_cffi1/recompiler.py +++ b/_cffi1/recompiler.py @@ -308,10 +308,13 @@ def _generate_cpy_typedef_decl(self, tp, name): pass - def _generate_cpy_typedef_ctx(self, tp, name): + def _typedef_ctx(self, tp, name): type_index = self._typesdict[tp] self._lsts["typename"].append( ' { "%s", %d },' % (name, type_index)) + + def _generate_cpy_typedef_ctx(self, tp, name): + self._typedef_ctx(tp, name) if getattr(tp, "origin", None) == "unknown_type": self._struct_ctx(tp, tp.name, approxname=None) elif isinstance(tp, model.NamedPointerType): @@ -521,8 +524,15 @@ raise NotImplementedError("internal inconsistency: %r is " "partial but was not seen at " "this point" % (tp,)) - assert tp.name.startswith('$') and tp.name[1:].isdigit() - self._struct_ctx(tp, None, tp.name[1:]) + if tp.name.startswith('$') and tp.name[1:].isdigit(): + approxname = tp.name[1:] + elif tp.name == '_IO_FILE' and tp.forcename == 'FILE': + approxname = 'FILE' + self._typedef_ctx(tp, 'FILE') + else: + raise NotImplementedError("internal inconsistency: %r" % + (tp,)) + self._struct_ctx(tp, None, approxname) def _fix_final_field_list(self, lst): count = 0 From noreply at buildbot.pypy.org Sun Apr 26 16:14:44 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 26 Apr 2015 16:14:44 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Don't emit _CFFI_F_CHECK_FIELDS on opaque structs (it is ignored anyway but confusing) Message-ID: <20150426141444.D1D401C0EC0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1849:8bb6d2089f42 Date: 2015-04-26 16:15 +0200 http://bitbucket.org/cffi/cffi/changeset/8bb6d2089f42/ Log: Don't emit _CFFI_F_CHECK_FIELDS on opaque structs (it is ignored anyway but confusing) diff --git a/_cffi1/recompiler.py b/_cffi1/recompiler.py --- a/_cffi1/recompiler.py +++ b/_cffi1/recompiler.py @@ -462,7 +462,9 @@ flags = [] if isinstance(tp, model.UnionType): flags.append("_CFFI_F_UNION") - if tp.partial or tp.has_anonymous_struct_fields(): + if tp.fldtypes is None: + pass # opaque + elif tp.partial or tp.has_anonymous_struct_fields(): pass # the field layout is obtained silently from the C compiler else: flags.append("_CFFI_F_CHECK_FIELDS") From noreply at buildbot.pypy.org Sun Apr 26 16:17:22 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 26 Apr 2015 16:17:22 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: complain clearly if an ffi.include() was used Message-ID: <20150426141722.AAD031C0EC0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1850:87eaba8629ad Date: 2015-04-26 16:17 +0200 http://bitbucket.org/cffi/cffi/changeset/87eaba8629ad/ Log: complain clearly if an ffi.include() was used diff --git a/_cffi1/recompiler.py b/_cffi1/recompiler.py --- a/_cffi1/recompiler.py +++ b/_cffi1/recompiler.py @@ -8,6 +8,9 @@ def __init__(self, ffi, module_name): self.ffi = ffi self.module_name = module_name + # + if ']' in self.ffi._cdefsources: + raise NotImplementedError("ffi.include()") def collect_type_table(self): self._typesdict = {} From noreply at buildbot.pypy.org Sun Apr 26 16:46:54 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 26 Apr 2015 16:46:54 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Move the demos from _cffi1/ back in the demo/ directory. Add the Message-ID: <20150426144654.1CB2F1C03F7@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1851:3a8606fab574 Date: 2015-04-26 16:47 +0200 http://bitbucket.org/cffi/cffi/changeset/3a8606fab574/ Log: Move the demos from _cffi1/ back in the demo/ directory. Add the cffi-1.0 version of the _curses demo too. diff --git a/_cffi1/bsdopendirtype.py b/_cffi1/bsdopendirtype.py deleted file mode 100644 --- a/_cffi1/bsdopendirtype.py +++ /dev/null @@ -1,48 +0,0 @@ -from _bsdopendirtype import ffi, lib - - -def _posix_error(): - raise OSError(ffi.errno, os.strerror(ffi.errno)) - -_dtype_to_smode = { - lib.DT_BLK: 0o060000, - lib.DT_CHR: 0o020000, - lib.DT_DIR: 0o040000, - lib.DT_FIFO: 0o010000, - lib.DT_LNK: 0o120000, - lib.DT_REG: 0o100000, - lib.DT_SOCK: 0o140000, -} - -def opendir(dir): - if len(dir) == 0: - dir = '.' - dirname = dir - if not dirname.endswith('/'): - dirname += '/' - dirp = lib.opendir(dir) - if dirp == ffi.NULL: - raise _posix_error() - try: - while True: - ffi.errno = 0 - dirent = lib.readdir(dirp) - if dirent == ffi.NULL: - if ffi.errno != 0: - raise _posix_error() - return - name = ffi.string(dirent.d_name) - if name == '.' or name == '..': - continue - name = dirname + name - try: - smode = _dtype_to_smode[dirent.d_type] - except KeyError: - smode = os.lstat(name).st_mode - yield name, smode - finally: - lib.closedir(dirp) - -if __name__ == '__main__': - for name, smode in opendir('/tmp'): - print hex(smode), name diff --git a/_cffi1/readdir2.py b/_cffi1/readdir2.py deleted file mode 100644 --- a/_cffi1/readdir2.py +++ /dev/null @@ -1,33 +0,0 @@ -# A Linux-only demo, using verify() instead of hard-coding the exact layouts -# -import sys -from _readdir2 import ffi, lib - -if not sys.platform.startswith('linux'): - raise Exception("Linux-only demo") - - -def walk(basefd, path): - print '{', path - dirfd = lib.openat(basefd, path, 0) - if dirfd < 0: - # error in openat() - return - dir = lib.fdopendir(dirfd) - dirent = ffi.new("struct dirent *") - result = ffi.new("struct dirent **") - while True: - if lib.readdir_r(dir, dirent, result): - # error in readdir_r() - break - if result[0] == ffi.NULL: - break - name = ffi.string(dirent.d_name) - print '%3d %s' % (dirent.d_type, name) - if dirent.d_type == lib.DT_DIR and name != '.' and name != '..': - walk(dirfd, name) - lib.closedir(dir) - print '}' - - -walk(-1, "/tmp") diff --git a/demo/_curses.py b/demo/_curses.py --- a/demo/_curses.py +++ b/demo/_curses.py @@ -1,265 +1,1075 @@ -"""Very partial replacement of the standard extension module '_curses'. -Just contains the minimal amount of stuff to make one of my curses -programs run. XXX should also check for and report errors. -""" -from cffi import FFI +"""Reimplementation of the standard extension module '_curses' using cffi.""" -ffi = FFI() +import sys +from functools import wraps +from _curses_cffi import ffi, lib -ffi.cdef(""" -typedef ... WINDOW; -typedef unsigned char bool; -typedef unsigned long chtype; -static const int ERR, OK; -WINDOW *initscr(void); -int endwin(void); -bool isendwin(void); - -const char *keyname(int c); -static const int KEY_MIN, KEY_MAX; - -int setupterm(char *term, int fildes, int *errret); - -int tigetflag(char *); -int tigetnum(char *); -char *tigetstr(char *); -char *tparm (const char *, ...); - -int cbreak(void); -int nocbreak(void); -int echo(void); -int noecho(void); -int keypad(WINDOW *win, bool bf); -int notimeout(WINDOW *win, bool bf); -void wtimeout(WINDOW *win, int delay); - -int def_prog_mode(void); -int def_shell_mode(void); -int reset_prog_mode(void); -int reset_shell_mode(void); -int resetty(void); -int savetty(void); -void getsyx(int y, int x); -void setsyx(int y, int x); -//int ripoffline(int line, int (*init)(WINDOW *, int)); -int curs_set(int visibility); -int napms(int ms); - -int start_color(void); -int init_pair(short pair, short f, short b); -int init_color(short color, short r, short g, short b); -bool has_colors(void); -bool can_change_color(void); -int color_content(short color, short *r, short *g, short *b); -int pair_content(short pair, short *f, short *b); - -int use_default_colors(void); - -static const int COLOR_BLACK; -static const int COLOR_RED; -static const int COLOR_GREEN; -static const int COLOR_YELLOW; -static const int COLOR_BLUE; -static const int COLOR_MAGENTA; -static const int COLOR_CYAN; -static const int COLOR_WHITE; - -static const int A_ATTRIBUTES; -static const int A_NORMAL; -static const int A_STANDOUT; -static const int A_UNDERLINE; -static const int A_REVERSE; -static const int A_BLINK; -static const int A_DIM; -static const int A_BOLD; -static const int A_ALTCHARSET; -static const int A_INVIS; -static const int A_PROTECT; -static const int A_CHARTEXT; -static const int A_COLOR; - -int COLORS, COLOR_PAIRS; - -void _m_getyx(WINDOW *win, int yx[2]); -void _m_getparyx(WINDOW *win, int yx[2]); -void _m_getbegyx(WINDOW *win, int yx[2]); -void _m_getmaxyx(WINDOW *win, int yx[2]); - -int wclear(WINDOW *win); -int wclrtoeol(WINDOW *win); -int wmove(WINDOW *win, int y, int x); -int waddstr(WINDOW *win, const char *str); -int mvwaddstr(WINDOW *win, int y, int x, const char *str); -void wbkgdset(WINDOW *win, chtype ch); -int wrefresh(WINDOW *win); -int wgetch(WINDOW *win); - -int getattrs(WINDOW *win); -int wattrset(WINDOW *win, int attrs); -""") - - -lib = ffi.verify(""" -#include -#include - -void _m_getyx(WINDOW *win, int yx[2]) { - getyx(win, yx[0], yx[1]); -} -void _m_getparyx(WINDOW *win, int yx[2]) { - getparyx(win, yx[0], yx[1]); -} -void _m_getbegyx(WINDOW *win, int yx[2]) { - getbegyx(win, yx[0], yx[1]); -} -void _m_getmaxyx(WINDOW *win, int yx[2]) { - getmaxyx(win, yx[0], yx[1]); -} -""", - libraries=['ncurses']) +def _copy_to_globals(name): + globals()[name] = getattr(lib, name) def _setup(): - globals().update(lib.__dict__) - for key in range(KEY_MIN, KEY_MAX): - key_n = keyname(key) - if key_n == ffi.NULL or ffi.string(key_n) == "UNKNOWN KEY": - continue - key_n = ffi.string(key_n).replace('(', '').replace(')', '') - globals()[key_n] = key + for name in ['ERR', 'OK', 'KEY_MIN', 'KEY_MAX', + 'A_ATTRIBUTES', 'A_NORMAL', 'A_STANDOUT', 'A_UNDERLINE', + 'A_REVERSE', 'A_BLINK', 'A_DIM', 'A_BOLD', 'A_ALTCHARSET', + 'A_PROTECT', 'A_CHARTEXT', 'A_COLOR', + 'COLOR_BLACK', 'COLOR_RED', 'COLOR_GREEN', 'COLOR_YELLOW', + 'COLOR_BLUE', 'COLOR_MAGENTA', 'COLOR_CYAN', 'COLOR_WHITE', + ]: + _copy_to_globals(name) + + if not lib._m_NetBSD: + _copy_to_globals('A_INVIS') + + for name in ['A_HORIZONTAL', 'A_LEFT', 'A_LOW', 'A_RIGHT', 'A_TOP', + 'A_VERTICAL', + ]: + if hasattr(lib, name): + _copy_to_globals(name) + + if lib._m_NCURSES_MOUSE_VERSION: + for name in ["BUTTON1_PRESSED", "BUTTON1_RELEASED", "BUTTON1_CLICKED", + "BUTTON1_DOUBLE_CLICKED", "BUTTON1_TRIPLE_CLICKED", + "BUTTON2_PRESSED", "BUTTON2_RELEASED", "BUTTON2_CLICKED", + "BUTTON2_DOUBLE_CLICKED", "BUTTON2_TRIPLE_CLICKED", + "BUTTON3_PRESSED", "BUTTON3_RELEASED", "BUTTON3_CLICKED", + "BUTTON3_DOUBLE_CLICKED", "BUTTON3_TRIPLE_CLICKED", + "BUTTON4_PRESSED", "BUTTON4_RELEASED", "BUTTON4_CLICKED", + "BUTTON4_DOUBLE_CLICKED", "BUTTON4_TRIPLE_CLICKED", + "BUTTON_SHIFT", "BUTTON_CTRL", "BUTTON_ALT", + "ALL_MOUSE_EVENTS", "REPORT_MOUSE_POSITION", + ]: + _copy_to_globals(name) + + if not lib._m_NetBSD: + for key in range(lib.KEY_MIN, lib.KEY_MAX): + key_n = lib.keyname(key) + if key_n == ffi.NULL: + continue + key_n = ffi.string(key_n) + if key_n == b"UNKNOWN KEY": + continue + if not isinstance(key_n, str): # python 3 + key_n = key_n.decode() + key_n = key_n.replace('(', '').replace(')', '') + globals()[key_n] = key _setup() +# Do we want this? +# version = "2.2" +# __version__ = "2.2" + + # ____________________________________________________________ + +_initialised_setupterm = False +_initialised = False +_initialised_color = False + + +def _ensure_initialised_setupterm(): + if not _initialised_setupterm: + raise error("must call (at least) setupterm() first") + + +def _ensure_initialised(): + if not _initialised: + raise error("must call initscr() first") + + +def _ensure_initialised_color(): + if not _initialised and _initialised_color: + raise error("must call start_color() first") + + +def _check_ERR(code, fname): + if code != lib.ERR: + return None + elif fname is None: + raise error("curses function returned ERR") + else: + raise error("%s() returned ERR" % (fname,)) + + +def _check_NULL(rval): + if rval == ffi.NULL: + raise error("curses function returned NULL") + return rval + + +def _call_lib(method_name, *args): + return getattr(lib, method_name)(*args) + + +def _call_lib_check_ERR(method_name, *args): + return _check_ERR(_call_lib(method_name, *args), method_name) + + +def _mk_no_return(method_name): + def _execute(): + _ensure_initialised() + return _call_lib_check_ERR(method_name) + _execute.__name__ = method_name + return _execute + + +def _mk_flag_func(method_name): + # This is in the CPython implementation, but not documented anywhere. + # We have to support it, though, even if it make me sad. + def _execute(flag=True): + _ensure_initialised() + if flag: + return _call_lib_check_ERR(method_name) + else: + return _call_lib_check_ERR('no' + method_name) + _execute.__name__ = method_name + return _execute + + +def _mk_return_val(method_name): + def _execute(): + return _call_lib(method_name) + _execute.__name__ = method_name + return _execute + + +def _mk_w_getyx(method_name): + def _execute(self): + y = _call_lib(method_name + 'y', self._win) + x = _call_lib(method_name + 'x', self._win) + return (y, x) + _execute.__name__ = method_name + return _execute + + +def _mk_w_no_return(method_name): + def _execute(self, *args): + return _call_lib_check_ERR(method_name, self._win, *args) + _execute.__name__ = method_name + return _execute + + +def _mk_w_return_val(method_name): + def _execute(self, *args): + return _call_lib(method_name, self._win, *args) + _execute.__name__ = method_name + return _execute + + +def _chtype(ch): + return int(ffi.cast("chtype", ch)) + +def _texttype(text): + if isinstance(text, str): + return text + elif isinstance(text, unicode): + return str(text) # default encoding + else: + raise TypeError("str or unicode expected, got a '%s' object" + % (type(text).__name__,)) + + +def _extract_yx(args): + if len(args) >= 2: + return (args[0], args[1], args[2:]) + return (None, None, args) + + +def _process_args(funcname, args, count, optcount, frontopt=0): + outargs = [] + if frontopt: + if len(args) > count + optcount: + # We have the front optional args here. + outargs.extend(args[:frontopt]) + args = args[frontopt:] + else: + # No front optional args, so make them None. + outargs.extend([None] * frontopt) + if (len(args) < count) or (len(args) > count + optcount): + raise error("%s requires %s to %s arguments" % ( + funcname, count, count + optcount + frontopt)) + outargs.extend(args) + return outargs + + +def _argspec(count, optcount=0, frontopt=0): + def _argspec_deco(func): + @wraps(func) + def _wrapped(self, *args): + outargs = _process_args( + func.__name__, args, count, optcount, frontopt) + return func(self, *outargs) + return _wrapped + return _argspec_deco + + +# ____________________________________________________________ + + class error(Exception): pass + class Window(object): - def __init__(self): - self._window = lib.initscr() + def __init__(self, window): + self._win = window - def getyx(self): - yx = ffi.new("int[2]") - lib._m_getyx(self._window, yx) - return tuple(yx) + def __del__(self): + if self._win != lib.stdscr: + lib.delwin(self._win) - def getparyx(self): - yx = ffi.new("int[2]") - lib._m_getparyx(self._window, yx) - return tuple(yx) + untouchwin = _mk_w_no_return("untouchwin") + touchwin = _mk_w_no_return("touchwin") + redrawwin = _mk_w_no_return("redrawwin") + insertln = _mk_w_no_return("winsertln") + erase = _mk_w_no_return("werase") + deleteln = _mk_w_no_return("wdeleteln") - def getbegyx(self): - yx = ffi.new("int[2]") - lib._m_getbegyx(self._window, yx) - return tuple(yx) + is_wintouched = _mk_w_return_val("is_wintouched") - def getmaxyx(self): - yx = ffi.new("int[2]") - lib._m_getmaxyx(self._window, yx) - return tuple(yx) + syncdown = _mk_w_return_val("wsyncdown") + syncup = _mk_w_return_val("wsyncup") + standend = _mk_w_return_val("wstandend") + standout = _mk_w_return_val("wstandout") + cursyncup = _mk_w_return_val("wcursyncup") + clrtoeol = _mk_w_return_val("wclrtoeol") + clrtobot = _mk_w_return_val("wclrtobot") + clear = _mk_w_return_val("wclear") - def addstr(self, *args): - y = None - attr = None - if len(args) == 1: - text, = args + idcok = _mk_w_no_return("idcok") + immedok = _mk_w_no_return("immedok") + timeout = _mk_w_no_return("wtimeout") + + getyx = _mk_w_getyx("getcur") + getbegyx = _mk_w_getyx("getbeg") + getmaxyx = _mk_w_getyx("getmax") + getparyx = _mk_w_getyx("getpar") + + clearok = _mk_w_no_return("clearok") + idlok = _mk_w_no_return("idlok") + leaveok = _mk_w_no_return("leaveok") + notimeout = _mk_w_no_return("notimeout") + scrollok = _mk_w_no_return("scrollok") + insdelln = _mk_w_no_return("winsdelln") + syncok = _mk_w_no_return("syncok") + + mvwin = _mk_w_no_return("mvwin") + mvderwin = _mk_w_no_return("mvderwin") + move = _mk_w_no_return("wmove") + + if not lib._m_STRICT_SYSV_CURSES: + resize = _mk_w_no_return("wresize") + + if lib._m_NetBSD: + keypad = _mk_w_return_val("keypad") + nodelay = _mk_w_return_val("nodelay") + else: + keypad = _mk_w_no_return("keypad") + nodelay = _mk_w_no_return("nodelay") + + @_argspec(1, 1, 2) + def addch(self, y, x, ch, attr=None): + if attr is None: + attr = lib.A_NORMAL + ch = _chtype(ch) + + if y is not None: + code = lib.mvwaddch(self._win, y, x, ch | attr) + else: + code = lib.waddch(self._win, ch | attr) + return _check_ERR(code, "addch") + + @_argspec(1, 1, 2) + def addstr(self, y, x, text, attr=None): + text = _texttype(text) + if attr is not None: + attr_old = lib.getattrs(self._win) + lib.wattrset(self._win, attr) + if y is not None: + code = lib.mvwaddstr(self._win, y, x, text) + else: + code = lib.waddstr(self._win, text) + if attr is not None: + lib.wattrset(self._win, attr_old) + return _check_ERR(code, "addstr") + + @_argspec(2, 1, 2) + def addnstr(self, y, x, text, n, attr=None): + text = _texttype(text) + if attr is not None: + attr_old = lib.getattrs(self._win) + lib.wattrset(self._win, attr) + if y is not None: + code = lib.mvwaddnstr(self._win, y, x, text, n) + else: + code = lib.waddnstr(self._win, text, n) + if attr is not None: + lib.wattrset(self._win, attr_old) + return _check_ERR(code, "addnstr") + + def bkgd(self, ch, attr=None): + if attr is None: + attr = lib.A_NORMAL + return _check_ERR(lib.wbkgd(self._win, _chtype(ch) | attr), "bkgd") + + attroff = _mk_w_no_return("wattroff") + attron = _mk_w_no_return("wattron") + attrset = _mk_w_no_return("wattrset") + + def bkgdset(self, ch, attr=None): + if attr is None: + attr = lib.A_NORMAL + lib.wbkgdset(self._win, _chtype(ch) | attr) + return None + + def border(self, ls=0, rs=0, ts=0, bs=0, tl=0, tr=0, bl=0, br=0): + lib.wborder(self._win, + _chtype(ls), _chtype(rs), _chtype(ts), _chtype(bs), + _chtype(tl), _chtype(tr), _chtype(bl), _chtype(br)) + return None + + def box(self, vertint=0, horint=0): + lib.box(self._win, vertint, horint) + return None + + @_argspec(1, 1, 2) + def chgat(self, y, x, num, attr=None): + # These optional args are in a weird order. + if attr is None: + attr = num + num = -1 + + color = ((attr >> 8) & 0xff) + attr = attr - (color << 8) + + if y is not None: + code = lib.mvwchgat(self._win, y, x, num, attr, color, ffi.NULL) + lib.touchline(self._win, y, 1) + else: + yy, _ = self.getyx() + code = lib.wchgat(self._win, num, attr, color, ffi.NULL) + lib.touchline(self._win, yy, 1) + return _check_ERR(code, "chgat") + + def delch(self, *args): + if len(args) == 0: + code = lib.wdelch(self._win) elif len(args) == 2: - text, attr = args - elif len(args) == 3: - y, x, text = args + code = lib.mvwdelch(self._win, *args) + else: + raise error("delch requires 0 or 2 arguments") + return _check_ERR(code, "[mv]wdelch") + + def derwin(self, *args): + nlines = 0 + ncols = 0 + if len(args) == 2: + begin_y, begin_x = args elif len(args) == 4: - y, x, text, attr = args + nlines, ncols, begin_y, begin_x = args else: - raise TypeError("addstr requires 1 to 4 arguments") + raise error("derwin requires 2 or 4 arguments") + + win = lib.derwin(self._win, nlines, ncols, begin_y, begin_x) + return Window(_check_NULL(win)) + + def echochar(self, ch, attr=None): + if attr is None: + attr = lib.A_NORMAL + ch = _chtype(ch) + + if lib._m_ispad(self._win): + code = lib.pechochar(self._win, ch | attr) + else: + code = lib.wechochar(self._win, ch | attr) + return _check_ERR(code, "echochar") + + if lib._m_NCURSES_MOUSE_VERSION: + enclose = _mk_w_return_val("wenclose") + + getbkgd = _mk_w_return_val("getbkgd") + + def getch(self, *args): + if len(args) == 0: + val = lib.wgetch(self._win) + elif len(args) == 2: + val = lib.mvwgetch(self._win, *args) + else: + raise error("getch requires 0 or 2 arguments") + return val + + def getkey(self, *args): + if len(args) == 0: + val = lib.wgetch(self._win) + elif len(args) == 2: + val = lib.mvwgetch(self._win, *args) + else: + raise error("getkey requires 0 or 2 arguments") + + if val == lib.ERR: + raise error("no input") + elif val <= 255: + return chr(val) + else: + # XXX: The following line is different if `__NetBSD__` is defined. + val = lib.keyname(val) + if val == ffi.NULL: + return "" + return ffi.string(val) + + @_argspec(0, 1, 2) + def getstr(self, y, x, n=1023): + n = min(n, 1023) + buf = ffi.new("char[1024]") # /* This should be big enough.. I hope */ + + if y is None: + val = lib.wgetnstr(self._win, buf, n) + else: + val = lib.mvwgetnstr(self._win, y, x, buf, n) + + if val == lib.ERR: + return "" + return ffi.string(buf) + + @_argspec(2, 1, 2) + def hline(self, y, x, ch, n, attr=None): + ch = _chtype(ch) + if attr is None: + attr = lib.A_NORMAL + if y is not None: + _check_ERR(lib.wmove(self._win, y, x), "wmove") + return _check_ERR(lib.whline(self._win, ch | attr, n), "hline") + + @_argspec(1, 1, 2) + def insch(self, y, x, ch, attr=None): + ch = _chtype(ch) + if attr is None: + attr = lib.A_NORMAL + if y is not None: + code = lib.mvwinsch(self._win, y, x, ch | attr) + else: + code = lib.winsch(self._win, ch | attr) + return _check_ERR(code, "insch") + + def inch(self, *args): + if len(args) == 0: + return lib.winch(self._win) + elif len(args) == 2: + return lib.mvwinch(self._win, *args) + else: + raise error("inch requires 0 or 2 arguments") + + @_argspec(0, 1, 2) + def instr(self, y, x, n=1023): + n = min(n, 1023) + buf = ffi.new("char[1024]") # /* This should be big enough.. I hope */ + if y is None: + code = lib.winnstr(self._win, buf, n) + else: + code = lib.mvwinnstr(self._win, y, x, buf, n) + + if code == lib.ERR: + return "" + return ffi.string(buf) + + @_argspec(1, 1, 2) + def insstr(self, y, x, text, attr=None): + text = _texttype(text) if attr is not None: - attr_old = getattrs(self._window) - wattrset(self._window, attr) + attr_old = lib.getattrs(self._win) + lib.wattrset(self._win, attr) if y is not None: - mvwaddstr(self._window, y, x, text) + code = lib.mvwinsstr(self._win, y, x, text) else: - waddstr(self._window, text) + code = lib.winsstr(self._win, text) if attr is not None: - wattrset(self._window, attr_old) + lib.wattrset(self._win, attr_old) + return _check_ERR(code, "insstr") - def bkgdset(self, bkgd, attr=A_NORMAL): - if isinstance(bkgd, str): - bkgd = ord(bkgd) - wbkgdset(self._window, bkgd | attr) + @_argspec(2, 1, 2) + def insnstr(self, y, x, text, n, attr=None): + text = _texttype(text) + if attr is not None: + attr_old = lib.getattrs(self._win) + lib.wattrset(self._win, attr) + if y is not None: + code = lib.mvwinsnstr(self._win, y, x, text, n) + else: + code = lib.winsnstr(self._win, text, n) + if attr is not None: + lib.wattrset(self._win, attr_old) + return _check_ERR(code, "insnstr") + def is_linetouched(self, line): + code = lib.is_linetouched(self._win, line) + if code == lib.ERR: + raise error("is_linetouched: line number outside of boundaries") + if code == lib.FALSE: + return False + return True - def _make_method(cname): - method = getattr(lib, cname) - def _execute(self, *args): - return method(self._window, *args) - return _execute + def noutrefresh(self, *args): + if lib._m_ispad(self._win): + if len(args) != 6: + raise error( + "noutrefresh() called for a pad requires 6 arguments") + return _check_ERR(lib.pnoutrefresh(self._win, *args), + "pnoutrefresh") + else: + # XXX: Better args check here? We need zero args. + return _check_ERR(lib.wnoutrefresh(self._win, *args), + "wnoutrefresh") - keypad = _make_method('keypad') - clear = _make_method('wclear') - clrtoeol = _make_method('wclrtoeol') - move = _make_method('wmove') - refresh = _make_method('wrefresh') - getch = _make_method('wgetch') - notimeout = _make_method('notimeout') - timeout = _make_method('wtimeout') + nooutrefresh = noutrefresh # "to be removed in 2.3", but in 2.7, 3.x. - del _make_method + def _copywin(self, dstwin, overlay, + sminr, sminc, dminr, dminc, dmaxr, dmaxc): + return _check_ERR(lib.copywin(self._win, dstwin._win, + sminr, sminc, dminr, dminc, dmaxr, dmaxc, + overlay), "copywin") + def overlay(self, dstwin, *args): + if len(args) == 6: + return self._copywin(dstwin, True, *args) + elif len(args) == 0: + return _check_ERR(lib.overlay(self._win, dstwin._win), "overlay") + else: + raise error("overlay requires one or seven arguments") -initscr = Window + def overwrite(self, dstwin, *args): + if len(args) == 6: + return self._copywin(dstwin, False, *args) + elif len(args) == 0: + return _check_ERR(lib.overwrite(self._win, dstwin._win), + "overwrite") + else: + raise error("overwrite requires one or seven arguments") -_setupterm_called = False + def putwin(self, filep): + # filestar = ffi.new("FILE *", filep) + return _check_ERR(lib.putwin(self._win, filep), "putwin") + def redrawln(self, beg, num): + return _check_ERR(lib.wredrawln(self._win, beg, num), "redrawln") -def _ensure_setupterm_called(): - if not _setupterm_called: - raise error("must call (at least) setupterm() first") + def refresh(self, *args): + if lib._m_ispad(self._win): + if len(args) != 6: + raise error( + "noutrefresh() called for a pad requires 6 arguments") + return _check_ERR(lib.prefresh(self._win, *args), "prefresh") + else: + # XXX: Better args check here? We need zero args. + return _check_ERR(lib.wrefresh(self._win, *args), "wrefresh") + + def setscrreg(self, y, x): + return _check_ERR(lib.wsetscrreg(self._win, y, x), "wsetscrreg") + + def subwin(self, *args): + nlines = 0 + ncols = 0 + if len(args) == 2: + begin_y, begin_x = args + elif len(args) == 4: + nlines, ncols, begin_y, begin_x = args + else: + raise error("subwin requires 2 or 4 arguments") + + if lib._m_ispad(self._win): + win = lib.subpad(self._win, nlines, ncols, begin_y, begin_x) + else: + win = lib.subwin(self._win, nlines, ncols, begin_y, begin_x) + return Window(_check_NULL(win)) + + def scroll(self, nlines=None): + if nlines is None: + return _check_ERR(lib.scroll(self._win), "scroll") + else: + return _check_ERR(lib.wscrl(self._win, nlines), "scroll") + + def touchline(self, st, cnt, val=None): + if val is None: + return _check_ERR(lib.touchline(self._win, st, cnt), "touchline") + else: + return _check_ERR(lib.wtouchln(self._win, st, cnt, val), + "touchline") + + @_argspec(2, 1, 2) + def vline(self, y, x, ch, n, attr=None): + ch = _chtype(ch) + if attr is None: + attr = lib.A_NORMAL + if y is not None: + _check_ERR(lib.wmove(self._win, y, x), "wmove") + return _check_ERR(lib.wvline(self._win, ch | attr, n), "vline") + + +beep = _mk_no_return("beep") +def_prog_mode = _mk_no_return("def_prog_mode") +def_shell_mode = _mk_no_return("def_shell_mode") +doupdate = _mk_no_return("doupdate") +endwin = _mk_no_return("endwin") +flash = _mk_no_return("flash") +nocbreak = _mk_no_return("nocbreak") +noecho = _mk_no_return("noecho") +nonl = _mk_no_return("nonl") +noraw = _mk_no_return("noraw") +reset_prog_mode = _mk_no_return("reset_prog_mode") +reset_shell_mode = _mk_no_return("reset_shell_mode") +resetty = _mk_no_return("resetty") +savetty = _mk_no_return("savetty") + +cbreak = _mk_flag_func("cbreak") +echo = _mk_flag_func("echo") +nl = _mk_flag_func("nl") +raw = _mk_flag_func("raw") + +baudrate = _mk_return_val("baudrate") +termattrs = _mk_return_val("termattrs") + +termname = _mk_return_val("termname") +longname = _mk_return_val("longname") + +can_change_color = _mk_return_val("can_change_color") +has_colors = _mk_return_val("has_colors") +has_ic = _mk_return_val("has_ic") +has_il = _mk_return_val("has_il") +isendwin = _mk_return_val("isendwin") +flushinp = _mk_return_val("flushinp") +noqiflush = _mk_return_val("noqiflush") + + +def filter(): + lib.filter() + return None + + +def color_content(color): + _ensure_initialised_color() + r, g, b = ffi.new("short *"), ffi.new("short *"), ffi.new("short *") + if lib.color_content(color, r, g, b) == lib.ERR: + raise error("Argument 1 was out of range. Check value of COLORS.") + return (r[0], g[0], b[0]) + + +def color_pair(n): + _ensure_initialised_color() + return (n << 8) + + +def curs_set(vis): + _ensure_initialised() + val = lib.curs_set(vis) + _check_ERR(val, "curs_set") + return val + + +def delay_output(ms): + _ensure_initialised() + return _check_ERR(lib.delay_output(ms), "delay_output") + + +def erasechar(): + _ensure_initialised() + return lib.erasechar() + + +def getsyx(): + _ensure_initialised() + yx = ffi.new("int[2]") + lib._m_getsyx(yx) + return (yx[0], yx[1]) + + +if lib._m_NCURSES_MOUSE_VERSION: + + def getmouse(): + _ensure_initialised() + mevent = ffi.new("MEVENT *") + _check_ERR(lib.getmouse(mevent), "getmouse") + return (mevent.id, mevent.x, mevent.y, mevent.z, mevent.bstate) + + def ungetmouse(id, x, y, z, bstate): + _ensure_initialised() + mevent = ffi.new("MEVENT *") + mevent.id, mevent.x, mevent.y, mevent.z, mevent.bstate = ( + id, x, y, z, bstate) + return _check_ERR(lib.ungetmouse(mevent), "ungetmouse") + + +def getwin(filep): + return Window(_check_NULL(lib.getwin(filep))) + + +def halfdelay(tenths): + _ensure_initialised() + return _check_ERR(lib.halfdelay(tenths), "halfdelay") + + +if not lib._m_STRICT_SYSV_CURSES: + def has_key(ch): + _ensure_initialised() + return lib.has_key(ch) + + +def init_color(color, r, g, b): + _ensure_initialised_color() + return _check_ERR(lib.init_color(color, r, g, b), "init_color") + + +def init_pair(pair, f, b): + _ensure_initialised_color() + return _check_ERR(lib.init_pair(pair, f, b), "init_pair") + + +def _mk_acs(name, ichar): + if len(ichar) == 1: + globals()[name] = lib.acs_map[ord(ichar)] + else: + globals()[name] = globals()[ichar] + + +def _map_acs(): + _mk_acs("ACS_ULCORNER", 'l') + _mk_acs("ACS_LLCORNER", 'm') + _mk_acs("ACS_URCORNER", 'k') + _mk_acs("ACS_LRCORNER", 'j') + _mk_acs("ACS_LTEE", 't') + _mk_acs("ACS_RTEE", 'u') + _mk_acs("ACS_BTEE", 'v') + _mk_acs("ACS_TTEE", 'w') + _mk_acs("ACS_HLINE", 'q') + _mk_acs("ACS_VLINE", 'x') + _mk_acs("ACS_PLUS", 'n') + _mk_acs("ACS_S1", 'o') + _mk_acs("ACS_S9", 's') + _mk_acs("ACS_DIAMOND", '`') + _mk_acs("ACS_CKBOARD", 'a') + _mk_acs("ACS_DEGREE", 'f') + _mk_acs("ACS_PLMINUS", 'g') + _mk_acs("ACS_BULLET", '~') + _mk_acs("ACS_LARROW", ',') + _mk_acs("ACS_RARROW", '+') + _mk_acs("ACS_DARROW", '.') + _mk_acs("ACS_UARROW", '-') + _mk_acs("ACS_BOARD", 'h') + _mk_acs("ACS_LANTERN", 'i') + _mk_acs("ACS_BLOCK", '0') + _mk_acs("ACS_S3", 'p') + _mk_acs("ACS_S7", 'r') + _mk_acs("ACS_LEQUAL", 'y') + _mk_acs("ACS_GEQUAL", 'z') + _mk_acs("ACS_PI", '{') + _mk_acs("ACS_NEQUAL", '|') + _mk_acs("ACS_STERLING", '}') + _mk_acs("ACS_BSSB", "ACS_ULCORNER") + _mk_acs("ACS_SSBB", "ACS_LLCORNER") + _mk_acs("ACS_BBSS", "ACS_URCORNER") + _mk_acs("ACS_SBBS", "ACS_LRCORNER") + _mk_acs("ACS_SBSS", "ACS_RTEE") + _mk_acs("ACS_SSSB", "ACS_LTEE") + _mk_acs("ACS_SSBS", "ACS_BTEE") + _mk_acs("ACS_BSSS", "ACS_TTEE") + _mk_acs("ACS_BSBS", "ACS_HLINE") + _mk_acs("ACS_SBSB", "ACS_VLINE") + _mk_acs("ACS_SSSS", "ACS_PLUS") + + +def initscr(): + if _initialised: + lib.wrefresh(lib.stdscr) + return Window(lib.stdscr) + + win = _check_NULL(lib.initscr()) + globals()['_initialised_setupterm'] = True + globals()['_initialised'] = True + + _map_acs() + + globals()["LINES"] = lib.LINES + globals()["COLS"] = lib.COLS + + return Window(win) def setupterm(term=None, fd=-1): + if fd == -1: + # XXX: Check for missing stdout here? + fd = sys.stdout.fileno() + + if _initialised_setupterm: + return None + if term is None: term = ffi.NULL - if fd < 0: - import sys - fd = sys.stdout.fileno() err = ffi.new("int *") - if lib.setupterm(term, fd, err) == ERR: - if err[0] == 0: - s = "setupterm: could not find terminal" - elif err[0] == 1: - s = "setupterm: could not find terminfo database" + if lib.setupterm(term, fd, err) == lib.ERR: + err = err[0] + if err == 0: + raise error("setupterm: could not find terminal") + elif err == -1: + raise error("setupterm: could not find terminfo database") else: - s = "setupterm: unknown error %d" % err[0] - raise error(s) - global _setupterm_called - _setupterm_called = True + raise error("setupterm: unknown error") + + globals()["_initialised_setupterm"] = True + return None + + +def intrflush(ch): + _ensure_initialised() + return _check_ERR(lib.intrflush(ffi.NULL, ch), "intrflush") + + +# XXX: #ifdef HAVE_CURSES_IS_TERM_RESIZED +def is_term_resized(lines, columns): + _ensure_initialised() + return lib.is_term_resized(lines, columns) + + +if not lib._m_NetBSD: + def keyname(ch): + _ensure_initialised() + if ch < 0: + raise error("invalid key number") + knp = lib.keyname(ch) + if knp == ffi.NULL: + return "" + return ffi.string(knp) + + +def killchar(): + return lib.killchar() + + +def meta(ch): + return _check_ERR(lib.meta(lib.stdscr, ch), "meta") + + +if lib._m_NCURSES_MOUSE_VERSION: + + def mouseinterval(interval): + _ensure_initialised() + return _check_ERR(lib.mouseinterval(interval), "mouseinterval") + + def mousemask(newmask): + _ensure_initialised() + oldmask = ffi.new("mmask_t *") + availmask = lib.mousemask(newmask, oldmask) + return (availmask, oldmask) + + +def napms(ms): + _ensure_initialised() + return lib.napms(ms) + + +def newpad(nlines, ncols): + _ensure_initialised() + return Window(_check_NULL(lib.newpad(nlines, ncols))) + + +def newwin(nlines, ncols, begin_y=None, begin_x=None): + _ensure_initialised() + if begin_x is None: + if begin_y is not None: + raise error("newwin requires 2 or 4 arguments") + begin_y = begin_x = 0 + + return Window(_check_NULL(lib.newwin(nlines, ncols, begin_y, begin_x))) + + +def pair_content(pair): + _ensure_initialised_color() + f = ffi.new("short *") + b = ffi.new("short *") + if lib.pair_content(pair, f, b) == lib.ERR: + raise error("Argument 1 was out of range. (1..COLOR_PAIRS-1)") + return (f, b) + + +def pair_number(pairvalue): + _ensure_initialised_color() + return (pairvalue & lib.A_COLOR) >> 8 + + +def putp(text): + text = _texttype(text) + return _check_ERR(lib.putp(text), "putp") + + +def qiflush(flag=True): + _ensure_initialised() + if flag: + lib.qiflush() + else: + lib.noqiflush() + return None + + +# XXX: Do something about the following? +# /* Internal helper used for updating curses.LINES, curses.COLS, _curses.LINES +# * and _curses.COLS */ +# #if defined(HAVE_CURSES_RESIZETERM) || defined(HAVE_CURSES_RESIZE_TERM) +# static int +# update_lines_cols(void) +# { +# PyObject *o; +# PyObject *m = PyImport_ImportModuleNoBlock("curses"); + +# if (!m) +# return 0; + +# o = PyInt_FromLong(LINES); +# if (!o) { +# Py_DECREF(m); +# return 0; +# } +# if (PyObject_SetAttrString(m, "LINES", o)) { +# Py_DECREF(m); +# Py_DECREF(o); +# return 0; +# } +# if (PyDict_SetItemString(ModDict, "LINES", o)) { +# Py_DECREF(m); +# Py_DECREF(o); +# return 0; +# } +# Py_DECREF(o); +# o = PyInt_FromLong(COLS); +# if (!o) { +# Py_DECREF(m); +# return 0; +# } +# if (PyObject_SetAttrString(m, "COLS", o)) { +# Py_DECREF(m); +# Py_DECREF(o); +# return 0; +# } +# if (PyDict_SetItemString(ModDict, "COLS", o)) { +# Py_DECREF(m); +# Py_DECREF(o); +# return 0; +# } +# Py_DECREF(o); +# Py_DECREF(m); +# return 1; +# } +# #endif + +# #ifdef HAVE_CURSES_RESIZETERM +# static PyObject * +# PyCurses_ResizeTerm(PyObject *self, PyObject *args) +# { +# int lines; +# int columns; +# PyObject *result; + +# PyCursesInitialised; + +# if (!PyArg_ParseTuple(args,"ii:resizeterm", &lines, &columns)) +# return NULL; + +# result = PyCursesCheckERR(resizeterm(lines, columns), "resizeterm"); +# if (!result) +# return NULL; +# if (!update_lines_cols()) +# return NULL; +# return result; +# } + +# #endif + +# #ifdef HAVE_CURSES_RESIZE_TERM +# static PyObject * +# PyCurses_Resize_Term(PyObject *self, PyObject *args) +# { +# int lines; +# int columns; + +# PyObject *result; + +# PyCursesInitialised; + +# if (!PyArg_ParseTuple(args,"ii:resize_term", &lines, &columns)) +# return NULL; + +# result = PyCursesCheckERR(resize_term(lines, columns), "resize_term"); +# if (!result) +# return NULL; +# if (!update_lines_cols()) +# return NULL; +# return result; +# } +# #endif /* HAVE_CURSES_RESIZE_TERM */ + + +def setsyx(y, x): + _ensure_initialised() + lib.setsyx(y, x) + return None + + +def start_color(): + _check_ERR(lib.start_color(), "start_color") + globals()["COLORS"] = lib.COLORS + globals()["COLOR_PAIRS"] = lib.COLOR_PAIRS + globals()["_initialised_color"] = True + return None def tigetflag(capname): - _ensure_setupterm_called() + _ensure_initialised_setupterm() return lib.tigetflag(capname) def tigetnum(capname): - _ensure_setupterm_called() + _ensure_initialised_setupterm() return lib.tigetnum(capname) def tigetstr(capname): - _ensure_setupterm_called() - out = lib.tigetstr(capname) - if out == ffi.NULL: + _ensure_initialised_setupterm() + val = lib.tigetstr(capname) + if int(ffi.cast("intptr_t", val)) in (0, -1): return None - return ffi.string(out) + return ffi.string(val) -def tparm(name, *args): - _ensure_setupterm_called() - cargs = [ffi.cast("long", arg) for arg in args] - return ffi.string(lib.tparm(name, *cargs)) +def tparm(fmt, i1=0, i2=0, i3=0, i4=0, i5=0, i6=0, i7=0, i8=0, i9=0): + args = [ffi.cast("int", i) for i in (i1, i2, i3, i4, i5, i6, i7, i8, i9)] + result = lib.tparm(fmt, *args) + if result == ffi.NULL: + raise error("tparm() returned NULL") + return ffi.string(result) -def color_pair(n): - return n << 8 +def typeahead(fd): + _ensure_initialised() + return _check_ERR(lib.typeahead(fd), "typeahead") + + +def unctrl(ch): + _ensure_initialised() + return lib.unctrl(_chtype(ch)) + + +def ungetch(ch): + _ensure_initialised() + return _check_ERR(lib.ungetch(_chtype(ch)), "ungetch") + + +def use_env(flag): + lib.use_env(flag) + return None + + +if not lib._m_STRICT_SYSV_CURSES: + + def use_default_colors(): + _ensure_initialised_color() + return _check_ERR(lib.use_default_colors(), "use_default_colors") diff --git a/demo/_curses_build.py b/demo/_curses_build.py new file mode 100644 --- /dev/null +++ b/demo/_curses_build.py @@ -0,0 +1,325 @@ +import sys +if sys.platform == 'win32': + #This module does not exist in windows + raise ImportError('No module named _curses') + +from cffi import FFI +from _cffi1 import recompile + +ffi = FFI() + +ffi.cdef(""" +typedef ... WINDOW; +typedef ... SCREEN; +typedef unsigned long mmask_t; +typedef unsigned char bool; +typedef unsigned long chtype; +typedef chtype attr_t; + +typedef struct +{ + short id; /* ID to distinguish multiple devices */ + int x, y, z; /* event coordinates (character-cell) */ + mmask_t bstate; /* button state bits */ +} +MEVENT; + +static const int ERR, OK; +static const int TRUE, FALSE; +static const int KEY_MIN, KEY_MAX; + +static const int COLOR_BLACK; +static const int COLOR_RED; +static const int COLOR_GREEN; +static const int COLOR_YELLOW; +static const int COLOR_BLUE; +static const int COLOR_MAGENTA; +static const int COLOR_CYAN; +static const int COLOR_WHITE; + +static const chtype A_ATTRIBUTES; +static const chtype A_NORMAL; +static const chtype A_STANDOUT; +static const chtype A_UNDERLINE; +static const chtype A_REVERSE; +static const chtype A_BLINK; +static const chtype A_DIM; +static const chtype A_BOLD; +static const chtype A_ALTCHARSET; +static const chtype A_INVIS; +static const chtype A_PROTECT; +static const chtype A_CHARTEXT; +static const chtype A_COLOR; + +static const int BUTTON1_RELEASED; +static const int BUTTON1_PRESSED; +static const int BUTTON1_CLICKED; +static const int BUTTON1_DOUBLE_CLICKED; +static const int BUTTON1_TRIPLE_CLICKED; +static const int BUTTON2_RELEASED; +static const int BUTTON2_PRESSED; +static const int BUTTON2_CLICKED; +static const int BUTTON2_DOUBLE_CLICKED; +static const int BUTTON2_TRIPLE_CLICKED; +static const int BUTTON3_RELEASED; +static const int BUTTON3_PRESSED; +static const int BUTTON3_CLICKED; +static const int BUTTON3_DOUBLE_CLICKED; +static const int BUTTON3_TRIPLE_CLICKED; +static const int BUTTON4_RELEASED; +static const int BUTTON4_PRESSED; +static const int BUTTON4_CLICKED; +static const int BUTTON4_DOUBLE_CLICKED; +static const int BUTTON4_TRIPLE_CLICKED; +static const int BUTTON_SHIFT; +static const int BUTTON_CTRL; +static const int BUTTON_ALT; +static const int ALL_MOUSE_EVENTS; +static const int REPORT_MOUSE_POSITION; + +int setupterm(char *, int, int *); + +WINDOW *stdscr; +int COLORS; +int COLOR_PAIRS; +int COLS; +int LINES; + +int baudrate(void); +int beep(void); +int box(WINDOW *, chtype, chtype); +bool can_change_color(void); +int cbreak(void); +int clearok(WINDOW *, bool); +int color_content(short, short*, short*, short*); +int copywin(const WINDOW*, WINDOW*, int, int, int, int, int, int, int); +int curs_set(int); +int def_prog_mode(void); +int def_shell_mode(void); +int delay_output(int); +int delwin(WINDOW *); +WINDOW * derwin(WINDOW *, int, int, int, int); +int doupdate(void); +int echo(void); +int endwin(void); +char erasechar(void); +void filter(void); +int flash(void); +int flushinp(void); +chtype getbkgd(WINDOW *); +WINDOW * getwin(FILE *); +int halfdelay(int); +bool has_colors(void); +bool has_ic(void); +bool has_il(void); +void idcok(WINDOW *, bool); +int idlok(WINDOW *, bool); +void immedok(WINDOW *, bool); +WINDOW * initscr(void); +int init_color(short, short, short, short); +int init_pair(short, short, short); +int intrflush(WINDOW *, bool); +bool isendwin(void); +bool is_linetouched(WINDOW *, int); +bool is_wintouched(WINDOW *); +const char * keyname(int); +int keypad(WINDOW *, bool); +char killchar(void); +int leaveok(WINDOW *, bool); +char * longname(void); +int meta(WINDOW *, bool); +int mvderwin(WINDOW *, int, int); +int mvwaddch(WINDOW *, int, int, const chtype); +int mvwaddnstr(WINDOW *, int, int, const char *, int); +int mvwaddstr(WINDOW *, int, int, const char *); +int mvwchgat(WINDOW *, int, int, int, attr_t, short, const void *); +int mvwdelch(WINDOW *, int, int); +int mvwgetch(WINDOW *, int, int); +int mvwgetnstr(WINDOW *, int, int, char *, int); +int mvwin(WINDOW *, int, int); +chtype mvwinch(WINDOW *, int, int); +int mvwinnstr(WINDOW *, int, int, char *, int); +int mvwinsch(WINDOW *, int, int, chtype); +int mvwinsnstr(WINDOW *, int, int, const char *, int); +int mvwinsstr(WINDOW *, int, int, const char *); +int napms(int); +WINDOW * newpad(int, int); +WINDOW * newwin(int, int, int, int); +int nl(void); +int nocbreak(void); +int nodelay(WINDOW *, bool); +int noecho(void); +int nonl(void); +void noqiflush(void); +int noraw(void); +int notimeout(WINDOW *, bool); +int overlay(const WINDOW*, WINDOW *); +int overwrite(const WINDOW*, WINDOW *); +int pair_content(short, short*, short*); +int pechochar(WINDOW *, const chtype); +int pnoutrefresh(WINDOW*, int, int, int, int, int, int); +int prefresh(WINDOW *, int, int, int, int, int, int); +int putwin(WINDOW *, FILE *); +void qiflush(void); +int raw(void); +int redrawwin(WINDOW *); +int resetty(void); +int reset_prog_mode(void); +int reset_shell_mode(void); +int savetty(void); +int scroll(WINDOW *); +int scrollok(WINDOW *, bool); +int start_color(void); +WINDOW * subpad(WINDOW *, int, int, int, int); +WINDOW * subwin(WINDOW *, int, int, int, int); +int syncok(WINDOW *, bool); +chtype termattrs(void); +char * termname(void); +int touchline(WINDOW *, int, int); +int touchwin(WINDOW *); +int typeahead(int); +int ungetch(int); +int untouchwin(WINDOW *); +void use_env(bool); +int waddch(WINDOW *, const chtype); +int waddnstr(WINDOW *, const char *, int); +int waddstr(WINDOW *, const char *); +int wattron(WINDOW *, int); +int wattroff(WINDOW *, int); +int wattrset(WINDOW *, int); +int wbkgd(WINDOW *, chtype); +void wbkgdset(WINDOW *, chtype); +int wborder(WINDOW *, chtype, chtype, chtype, chtype, + chtype, chtype, chtype, chtype); +int wchgat(WINDOW *, int, attr_t, short, const void *); +int wclear(WINDOW *); +int wclrtobot(WINDOW *); +int wclrtoeol(WINDOW *); +void wcursyncup(WINDOW *); +int wdelch(WINDOW *); +int wdeleteln(WINDOW *); +int wechochar(WINDOW *, const chtype); +int werase(WINDOW *); +int wgetch(WINDOW *); +int wgetnstr(WINDOW *, char *, int); +int whline(WINDOW *, chtype, int); +chtype winch(WINDOW *); +int winnstr(WINDOW *, char *, int); +int winsch(WINDOW *, chtype); +int winsdelln(WINDOW *, int); +int winsertln(WINDOW *); +int winsnstr(WINDOW *, const char *, int); +int winsstr(WINDOW *, const char *); +int wmove(WINDOW *, int, int); +int wresize(WINDOW *, int, int); +int wnoutrefresh(WINDOW *); +int wredrawln(WINDOW *, int, int); +int wrefresh(WINDOW *); +int wscrl(WINDOW *, int); +int wsetscrreg(WINDOW *, int, int); +int wstandout(WINDOW *); +int wstandend(WINDOW *); +void wsyncdown(WINDOW *); +void wsyncup(WINDOW *); +void wtimeout(WINDOW *, int); +int wtouchln(WINDOW *, int, int, int); +int wvline(WINDOW *, chtype, int); +int tigetflag(char *); +int tigetnum(char *); +char * tigetstr(char *); +int putp(const char *); +char * tparm(const char *, ...); +int getattrs(const WINDOW *); +int getcurx(const WINDOW *); +int getcury(const WINDOW *); +int getbegx(const WINDOW *); +int getbegy(const WINDOW *); +int getmaxx(const WINDOW *); +int getmaxy(const WINDOW *); +int getparx(const WINDOW *); +int getpary(const WINDOW *); + +int getmouse(MEVENT *); +int ungetmouse(MEVENT *); +mmask_t mousemask(mmask_t, mmask_t *); +bool wenclose(const WINDOW *, int, int); +int mouseinterval(int); + +void setsyx(int y, int x); +const char *unctrl(chtype); +int use_default_colors(void); + +int has_key(int); +bool is_term_resized(int, int); + +#define _m_STRICT_SYSV_CURSES ... +#define _m_NCURSES_MOUSE_VERSION ... +#define _m_NetBSD ... +int _m_ispad(WINDOW *); + +chtype acs_map[]; + +// For _curses_panel: + +typedef ... PANEL; + +WINDOW *panel_window(const PANEL *); +void update_panels(void); +int hide_panel(PANEL *); +int show_panel(PANEL *); +int del_panel(PANEL *); +int top_panel(PANEL *); +int bottom_panel(PANEL *); +PANEL *new_panel(WINDOW *); +PANEL *panel_above(const PANEL *); +PANEL *panel_below(const PANEL *); +int set_panel_userptr(PANEL *, void *); +const void *panel_userptr(const PANEL *); +int move_panel(PANEL *, int, int); +int replace_panel(PANEL *,WINDOW *); +int panel_hidden(const PANEL *); + +void _m_getsyx(int *yx); +""") + + +recompile(ffi, "_curses_cffi", """ +#ifdef __APPLE__ +/* the following define is necessary for OS X 10.6+; without it, the + Apple-supplied ncurses.h sets NCURSES_OPAQUE to 1, and then Python + can't get at the WINDOW flags field. */ +#define NCURSES_OPAQUE 0 +#endif + +#include +#include +#include + +#if defined STRICT_SYSV_CURSES +#define _m_STRICT_SYSV_CURSES TRUE +#else +#define _m_STRICT_SYSV_CURSES FALSE +#endif + +#if defined NCURSES_MOUSE_VERSION +#define _m_NCURSES_MOUSE_VERSION TRUE +#else +#define _m_NCURSES_MOUSE_VERSION FALSE +#endif + +#if defined __NetBSD__ +#define _m_NetBSD TRUE +#else +#define _m_NetBSD FALSE +#endif + +int _m_ispad(WINDOW *win) { + // may not have _flags (and possibly _ISPAD), + // but for now let's assume that always has it + return (win->_flags & _ISPAD); +} + +void _m_getsyx(int *yx) { + getsyx(yx[0], yx[1]); +} +""", libraries=['ncurses', 'panel']) diff --git a/_cffi1/bsdopendirtype_build.py b/demo/bsdopendirtype_build.py rename from _cffi1/bsdopendirtype_build.py rename to demo/bsdopendirtype_build.py diff --git a/demo/readdir2.py b/demo/readdir2.py --- a/demo/readdir2.py +++ b/demo/readdir2.py @@ -1,65 +1,32 @@ # A Linux-only demo, using verify() instead of hard-coding the exact layouts # import sys -from cffi import FFI +from _readdir2 import ffi, lib if not sys.platform.startswith('linux'): raise Exception("Linux-only demo") -ffi = FFI() -ffi.cdef(""" - - typedef ... DIR; - - struct dirent { - unsigned char d_type; /* type of file; not supported - by all file system types */ - char d_name[...]; /* filename */ - ...; - }; - - int readdir_r(DIR *dirp, struct dirent *entry, struct dirent **result); - int openat(int dirfd, const char *pathname, int flags); - DIR *fdopendir(int fd); - int closedir(DIR *dirp); - - static const int DT_DIR; - -""") -ffi.C = ffi.verify(""" -#ifndef _ATFILE_SOURCE -# define _ATFILE_SOURCE -#endif -#ifndef _BSD_SOURCE -# define _BSD_SOURCE -#endif -#include -#include -#include -""") - - def walk(basefd, path): print '{', path - dirfd = ffi.C.openat(basefd, path, 0) + dirfd = lib.openat(basefd, path, 0) if dirfd < 0: # error in openat() return - dir = ffi.C.fdopendir(dirfd) + dir = lib.fdopendir(dirfd) dirent = ffi.new("struct dirent *") result = ffi.new("struct dirent **") while True: - if ffi.C.readdir_r(dir, dirent, result): + if lib.readdir_r(dir, dirent, result): # error in readdir_r() break if result[0] == ffi.NULL: break name = ffi.string(dirent.d_name) print '%3d %s' % (dirent.d_type, name) - if dirent.d_type == ffi.C.DT_DIR and name != '.' and name != '..': + if dirent.d_type == lib.DT_DIR and name != '.' and name != '..': walk(dirfd, name) - ffi.C.closedir(dir) + lib.closedir(dir) print '}' diff --git a/_cffi1/readdir2_build.py b/demo/readdir2_build.py rename from _cffi1/readdir2_build.py rename to demo/readdir2_build.py From noreply at buildbot.pypy.org Sun Apr 26 18:10:39 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 26 Apr 2015 18:10:39 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Support dotted names in 'module_name' Message-ID: <20150426161039.300731C0EC0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1852:288228787b93 Date: 2015-04-26 18:11 +0200 http://bitbucket.org/cffi/cffi/changeset/288228787b93/ Log: Support dotted names in 'module_name' diff --git a/_cffi1/recompiler.py b/_cffi1/recompiler.py --- a/_cffi1/recompiler.py +++ b/_cffi1/recompiler.py @@ -209,8 +209,9 @@ prnt() # # the init function, loading _cffi_backend and calling a method there + base_module_name = self.module_name.split('.')[-1] prnt('PyMODINIT_FUNC') - prnt('init%s(void)' % (self.module_name,)) + prnt('init%s(void)' % (base_module_name,)) prnt('{') prnt(' if (_cffi_init() < 0)') prnt(' return;') diff --git a/_cffi1/test_recompiler.py b/_cffi1/test_recompiler.py --- a/_cffi1/test_recompiler.py +++ b/_cffi1/test_recompiler.py @@ -1,6 +1,7 @@ -import sys, py +import sys, os, py from cffi import FFI, VerificationError from _cffi1 import recompiler +from _cffi1.udir import udir def check_type_table(input, expected_output): @@ -400,3 +401,22 @@ assert ffi1.typeof(name) is not ffi2.typeof(name) # sanity check: twice 'ffi1' assert ffi1.typeof("struct foo_s*") is ffi1.typeof("struct foo_s *") + +def test_module_name_in_package(): + ffi = FFI() + ffi.cdef("int foo(int);") + recompiler.recompile(ffi, "test_module_name_in_package.mymod", + "int foo(int x) { return x + 32; }", + tmpdir=str(udir)) + old_sys_path = sys.path[:] + try: + package_dir = udir.join('test_module_name_in_package') + assert os.path.isdir(str(package_dir)) + assert len(os.listdir(str(package_dir))) > 0 + package_dir.join('__init__.py').write('') + # + sys.path.insert(0, str(udir)) + import test_module_name_in_package.mymod + assert test_module_name_in_package.mymod.lib.foo(10) == 42 + finally: + sys.path[:] = old_sys_path From noreply at buildbot.pypy.org Sun Apr 26 18:36:54 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 26 Apr 2015 18:36:54 +0200 (CEST) Subject: [pypy-commit] cffi default: Obscure workaround for https://bugs.python.org/issue23246 on Python 2.7.9 Message-ID: <20150426163654.82DFE1C116F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1853:ee5aa6781207 Date: 2015-04-26 18:37 +0200 http://bitbucket.org/cffi/cffi/changeset/ee5aa6781207/ Log: Obscure workaround for https://bugs.python.org/issue23246 on Python 2.7.9 diff --git a/cffi/ffiplatform.py b/cffi/ffiplatform.py --- a/cffi/ffiplatform.py +++ b/cffi/ffiplatform.py @@ -1,4 +1,4 @@ -import os +import sys, os class VerificationError(Exception): @@ -14,6 +14,15 @@ LIST_OF_FILE_NAMES = ['sources', 'include_dirs', 'library_dirs', 'extra_objects', 'depends'] +def _hack_at_distutils(): + # Windows-only workaround for some configurations: see + # https://bugs.python.org/issue23246 (Python 2.7.9) + if sys.platform == "win32": + try: + import setuptools # for side-effects, patches distutils + except ImportError: + pass + def get_extension(srcfilename, modname, sources=(), **kwds): from distutils.core import Extension allsources = [srcfilename] @@ -37,6 +46,7 @@ def _build(tmpdir, ext): # XXX compact but horrible :-( + _hack_at_distutils() from distutils.core import Distribution import distutils.errors # From noreply at buildbot.pypy.org Sun Apr 26 18:39:59 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 26 Apr 2015 18:39:59 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: hg merge default Message-ID: <20150426163959.665951C0359@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1854:b20650f90e1c Date: 2015-04-26 18:40 +0200 http://bitbucket.org/cffi/cffi/changeset/b20650f90e1c/ Log: hg merge default diff --git a/_cffi1/test_new_ffi_1.py b/_cffi1/test_new_ffi_1.py --- a/_cffi1/test_new_ffi_1.py +++ b/_cffi1/test_new_ffi_1.py @@ -60,6 +60,10 @@ struct { int a, b; }; union { int c, d; }; }; + struct nested_field_ofs_s { + struct { int a; char b; }; + union { char c; }; + }; union nested_anon_u { struct { int a, b; }; union { int c, d; }; @@ -81,7 +85,7 @@ ffi = module.ffi -class TestOldFFI1: +class TestNewFFI1: def test_integer_ranges(self): for (c_type, size) in [('char', 1), @@ -1328,6 +1332,14 @@ assert p.c == 14 assert p.d == 14 + def test_nested_field_offset_align(self): + # struct nested_field_ofs_s { + # struct { int a; char b; }; + # union { char c; }; + # }; + assert ffi.offsetof("struct nested_field_ofs_s", "c") == 2 * SIZE_OF_INT + assert ffi.sizeof("struct nested_field_ofs_s") == 3 * SIZE_OF_INT + def test_nested_anonymous_union(self): # union nested_anon_u { # struct { int a, b; }; diff --git a/cffi/ffiplatform.py b/cffi/ffiplatform.py --- a/cffi/ffiplatform.py +++ b/cffi/ffiplatform.py @@ -1,4 +1,4 @@ -import os +import sys, os class VerificationError(Exception): @@ -14,6 +14,15 @@ LIST_OF_FILE_NAMES = ['sources', 'include_dirs', 'library_dirs', 'extra_objects', 'depends'] +def _hack_at_distutils(): + # Windows-only workaround for some configurations: see + # https://bugs.python.org/issue23246 (Python 2.7.9) + if sys.platform == "win32": + try: + import setuptools # for side-effects, patches distutils + except ImportError: + pass + def get_extension(srcfilename, modname, sources=(), **kwds): from distutils.core import Extension allsources = [srcfilename] @@ -37,6 +46,7 @@ def _build(tmpdir, ext): # XXX compact but horrible :-( + _hack_at_distutils() from distutils.core import Distribution import distutils.errors # diff --git a/testing/backend_tests.py b/testing/backend_tests.py --- a/testing/backend_tests.py +++ b/testing/backend_tests.py @@ -1388,6 +1388,17 @@ assert p.c == 14 assert p.d == 14 + def test_nested_field_offset_align(self): + ffi = FFI(backend=self.Backend()) + ffi.cdef(""" + struct foo_s { + struct { int a; char b; }; + union { char c; }; + }; + """) + assert ffi.offsetof("struct foo_s", "c") == 2 * SIZE_OF_INT + assert ffi.sizeof("struct foo_s") == 3 * SIZE_OF_INT + def test_nested_anonymous_union(self): ffi = FFI(backend=self.Backend()) ffi.cdef(""" diff --git a/testing/test_ctypes.py b/testing/test_ctypes.py --- a/testing/test_ctypes.py +++ b/testing/test_ctypes.py @@ -28,6 +28,9 @@ def test_nested_anonymous_struct(self): py.test.skip("ctypes backend: not supported: nested anonymous struct") + def test_nested_field_offset_align(self): + py.test.skip("ctypes backend: not supported: nested anonymous struct") + def test_nested_anonymous_union(self): py.test.skip("ctypes backend: not supported: nested anonymous union") diff --git a/testing/test_verify.py b/testing/test_verify.py --- a/testing/test_verify.py +++ b/testing/test_verify.py @@ -1229,11 +1229,11 @@ xxx def test_opaque_integer_as_function_result(): - import platform - if platform.machine().startswith('sparc'): - py.test.skip('Breaks horribly on sparc (SIGILL + corrupted stack)') - elif platform.machine() == 'mips64' and sys.maxsize > 2**32: - py.test.skip('Segfaults on mips64el') + #import platform + #if platform.machine().startswith('sparc'): + # py.test.skip('Breaks horribly on sparc (SIGILL + corrupted stack)') + #elif platform.machine() == 'mips64' and sys.maxsize > 2**32: + # py.test.skip('Segfaults on mips64el') # XXX bad abuse of "struct { ...; }". It only works a bit by chance # anyway. XXX think about something better :-( ffi = FFI() From noreply at buildbot.pypy.org Sun Apr 26 19:29:51 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 26 Apr 2015 19:29:51 +0200 (CEST) Subject: [pypy-commit] pypy default: Try to fix ARM Message-ID: <20150426172951.32A0E1C0A5B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76939:61817d7f51b1 Date: 2015-04-26 19:29 +0200 http://bitbucket.org/pypy/pypy/changeset/61817d7f51b1/ Log: Try to fix ARM diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -459,6 +459,8 @@ return fcond emit_op_jit_debug = emit_op_debug_merge_point emit_op_keepalive = emit_op_debug_merge_point + emit_op_enter_portal_frame = emit_op_debug_merge_point + emit_op_leave_portal_frame = emit_op_debug_merge_point def emit_op_cond_call_gc_wb(self, op, arglocs, regalloc, fcond): self._write_barrier_fastpath(self.mc, op.getdescr(), arglocs, fcond) diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -1155,6 +1155,8 @@ prepare_op_debug_merge_point = void prepare_op_jit_debug = void prepare_op_keepalive = void + prepare_op_enter_portal_frame = void + prepare_op_leave_portal_frame = void def prepare_op_cond_call_gc_wb(self, op, fcond): assert op.result is None From noreply at buildbot.pypy.org Sun Apr 26 19:47:24 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 26 Apr 2015 19:47:24 +0200 (CEST) Subject: [pypy-commit] pypy default: If cffi is not installed, just skip this test Message-ID: <20150426174724.56F5E1C0845@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76940:9c621f634f52 Date: 2015-04-26 19:47 +0200 http://bitbucket.org/pypy/pypy/changeset/9c621f634f52/ Log: If cffi is not installed, just skip this test diff --git a/rpython/jit/backend/llsupport/test/test_skiplist.py b/rpython/jit/backend/llsupport/test/test_skiplist.py --- a/rpython/jit/backend/llsupport/test/test_skiplist.py +++ b/rpython/jit/backend/llsupport/test/test_skiplist.py @@ -1,5 +1,9 @@ import random, os -import cffi +try: + import cffi +except ImportError: + import py + py.test.skip("cffi not installed") ffi = cffi.FFI() From noreply at buildbot.pypy.org Sun Apr 26 21:22:56 2015 From: noreply at buildbot.pypy.org (mattip) Date: Sun, 26 Apr 2015 21:22:56 +0200 (CEST) Subject: [pypy-commit] pypy default: (ab)use link_files since it appears _after_ the object files and gcc's linker is picky about order Message-ID: <20150426192256.B71981C03F7@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76941:8bbba5f3aeb8 Date: 2015-04-26 22:23 +0300 http://bitbucket.org/pypy/pypy/changeset/8bbba5f3aeb8/ Log: (ab)use link_files since it appears _after_ the object files and gcc's linker is picky about order diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py --- a/pypy/module/_vmprof/interp_vmprof.py +++ b/pypy/module/_vmprof/interp_vmprof.py @@ -27,7 +27,7 @@ include_dirs = [SRC], includes = ['vmprof.h', 'trampoline.h'], separate_module_files = [SRC.join('trampoline.asmgcc.s')], - link_extra = ['-Wl,-Bstatic', '-lunwind', '-Wl,-Bdynamic', '-llzma'], + link_files = ['-Wl,-Bstatic', '-lunwind', '-Wl,-Bdynamic', '-llzma'], post_include_bits=[""" void pypy_vmprof_init(void); From noreply at buildbot.pypy.org Sun Apr 26 23:26:02 2015 From: noreply at buildbot.pypy.org (mattip) Date: Sun, 26 Apr 2015 23:26:02 +0200 (CEST) Subject: [pypy-commit] pypy numpy-fixes: test, redo complex floordiv to avoid overflow if possible Message-ID: <20150426212602.521721C0A5B@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: numpy-fixes Changeset: r76942:9ea844c184eb Date: 2015-04-26 23:33 +0300 http://bitbucket.org/pypy/pypy/changeset/9ea844c184eb/ Log: test, redo complex floordiv to avoid overflow if possible diff --git a/pypy/module/micronumpy/test/test_complex.py b/pypy/module/micronumpy/test/test_complex.py --- a/pypy/module/micronumpy/test/test_complex.py +++ b/pypy/module/micronumpy/test/test_complex.py @@ -486,12 +486,16 @@ assert c[i] == max(a[i], b[i]) - def test_abs_overflow(self): - from numpy import array, absolute, isinf + def test_complex_overflow(self): + from numpy import array, absolute, isinf, complex128, floor_divide a = array(complex(1.5e308,1.5e308)) # Prints a RuntimeWarning, but does not raise b = absolute(a) assert isinf(b) + c = array([1.e+110, 1.e-110], dtype=complex128) + d = floor_divide(c**2, c) + assert (d == [1.e+110, 0]).all() + def test_basic(self): @@ -598,7 +602,7 @@ # but numpy.raises a TypeError if '__pypy__' in sys.builtin_module_names: exct, excm = TypeError, 'readonly attribute' - else: + else : exct, excm = AttributeError, 'is not writable' exc = raises(exct, 'c2.real = 10.') assert excm in exc.value[0] From noreply at buildbot.pypy.org Sun Apr 26 23:34:29 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 26 Apr 2015 23:34:29 +0200 (CEST) Subject: [pypy-commit] pypy default: The "-llzma" seems to have been checked in by accident. Removing it seems to have no bad effect, and liblzma is not Message-ID: <20150426213429.832A51C116B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76943:b43cd584adec Date: 2015-04-26 23:02 +0200 http://bitbucket.org/pypy/pypy/changeset/b43cd584adec/ Log: The "-llzma" seems to have been checked in by accident. Removing it seems to have no bad effect, and liblzma is not necessarily installed (e.g. not documented in the list of deps) diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py --- a/pypy/module/_vmprof/interp_vmprof.py +++ b/pypy/module/_vmprof/interp_vmprof.py @@ -27,7 +27,7 @@ include_dirs = [SRC], includes = ['vmprof.h', 'trampoline.h'], separate_module_files = [SRC.join('trampoline.asmgcc.s')], - link_files = ['-Wl,-Bstatic', '-lunwind', '-Wl,-Bdynamic', '-llzma'], + link_files = ['-Wl,-Bstatic', '-lunwind', '-Wl,-Bdynamic'], post_include_bits=[""" void pypy_vmprof_init(void); From noreply at buildbot.pypy.org Sun Apr 26 23:59:26 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 26 Apr 2015 23:59:26 +0200 (CEST) Subject: [pypy-commit] pypy default: disable _vmprof Message-ID: <20150426215926.E5C621C116F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76944:d32226e955cb Date: 2015-04-26 23:59 +0200 http://bitbucket.org/pypy/pypy/changeset/d32226e955cb/ Log: disable _vmprof diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -39,6 +39,7 @@ ]) if sys.platform.startswith('linux') and sys.maxint > 2147483647: + if 0: # XXX disabled until we fix the absurd .so mess working_modules.add('_vmprof') translation_modules = default_modules.copy() From noreply at buildbot.pypy.org Mon Apr 27 06:44:46 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Mon, 27 Apr 2015 06:44:46 +0200 (CEST) Subject: [pypy-commit] pypy can_cast: Implement enough of can_cast(, dtype) to pass the test Message-ID: <20150427044446.AF1291C0EC0@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: can_cast Changeset: r76945:d45abfb9685d Date: 2015-04-27 05:44 +0100 http://bitbucket.org/pypy/pypy/changeset/d45abfb9685d/ Log: Implement enough of can_cast(, dtype) to pass the test diff --git a/pypy/module/micronumpy/arrayops.py b/pypy/module/micronumpy/arrayops.py --- a/pypy/module/micronumpy/arrayops.py +++ b/pypy/module/micronumpy/arrayops.py @@ -10,6 +10,7 @@ from .boxes import W_GenericBox from .types import ( Bool, ULong, Long, Float64, Complex64, UnicodeType, VoidType, ObjectType) +from .descriptor import get_dtype_cache def where(space, w_arr, w_x=None, w_y=None): @@ -360,7 +361,11 @@ return can_cast_type(space, from_type, target, casting) if not from_type.is_native(): value = value.descr_byteswap(space) - return can_cast_type(space, from_type, target, casting) # XXX: stub impl + dtypenum, altnum = value.min_dtype() + if target.is_unsigned(): + dtypenum = altnum + dtype = get_dtype_cache(space).dtypes_by_num[dtypenum] + return can_cast_type(space, dtype, target, casting) # XXX: stub impl def is_scalar_w(space, w_arg): return (isinstance(w_arg, W_GenericBox) or diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py --- a/pypy/module/micronumpy/boxes.py +++ b/pypy/module/micronumpy/boxes.py @@ -874,4 +874,3 @@ __new__ = interp2app(W_ObjectBox.descr__new__.im_func), __getattr__ = interp2app(W_ObjectBox.descr__getattr__), ) - diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -1,5 +1,6 @@ import functools import math +from rpython.rlib.unroll import unrolling_iterable from pypy.interpreter.error import OperationError, oefmt from pypy.objspace.std.floatobject import float2string from pypy.objspace.std.complexobject import str_format @@ -2472,3 +2473,56 @@ for tp2 in complex_types: if tp1.basesize() <= tp2.basesize(): enable_cast(tp1, tp2) + +_int_types = [(Int8, UInt8), (Int16, UInt16), (Int32, UInt32), + (Int64, UInt64), (Long, ULong)] +for Int_t, UInt_t in _int_types: + Int_t.Unsigned = UInt_t + UInt_t.Signed = Int_t + size = rffi.sizeof(Int_t.T) + Int_t.min_value = rffi.cast(Int_t.T, -1) << (8*size - 1) + Int_t.max_value = ~Int_t.min_value + UInt_t.max_value = ~rffi.cast(UInt_t.T, 0) + + +signed_types = [Int8, Int16, Int32, Int64, Long] + +for Int_t in signed_types: + UInt_t = Int_t.Unsigned + smaller_types = [tp for tp in signed_types + if rffi.sizeof(tp.T) < rffi.sizeof(Int_t.T)] + smaller_types = unrolling_iterable( + [(tp, tp.Unsigned) for tp in smaller_types]) + def min_dtype(self): + for Small, USmall in smaller_types: + signed_max = rffi.cast(UInt_t.T, Small.max_value) + unsigned_max = rffi.cast(UInt_t.T, USmall.max_value) + if self.value <= unsigned_max: + if self.value <= signed_max: + return Small.num, USmall.num + else: + return USmall.num, USmall.num + if self.value <= rffi.cast(UInt_t.T, Int_t.max_value): + return Int_t.num, UInt_t.num + else: + return UInt_t.num, UInt_t.num + UInt_t.BoxType.min_dtype = min_dtype + + def min_dtype(self): + if self.value >= 0: + for Small, USmall in smaller_types: + signed_max = rffi.cast(UInt_t.T, Small.max_value) + unsigned_max = rffi.cast(UInt_t.T, USmall.max_value) + if self.value <= unsigned_max: + if self.value <= signed_max: + return Small.num, USmall.num + else: + return USmall.num, USmall.num + return Int_t.num, UInt_t.num + else: + for Small, USmall in smaller_types: + signed_min = rffi.cast(UInt_t.T, Small.min_value) + if self.value >= signed_max: + return Small.num, Small.num + return Int_t.num, Int_t.num + Int_t.BoxType.min_dtype = min_dtype From noreply at buildbot.pypy.org Mon Apr 27 10:11:55 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 27 Apr 2015 10:11:55 +0200 (CEST) Subject: [pypy-commit] pypy.org extradoc: Try to avoid confusion Message-ID: <20150427081155.951211C04A4@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r598:88189d3b1d8f Date: 2015-04-27 10:12 +0200 http://bitbucket.org/pypy/pypy.org/changeset/88189d3b1d8f/ Log: Try to avoid confusion diff --git a/features.html b/features.html --- a/features.html +++ b/features.html @@ -145,6 +145,7 @@

 cd pypy/sandbox
 pypy_interact.py path/to/pypy-sandbox
+# don't confuse it with pypy/goal/pyinteractive.py!

You get a fully sandboxed interpreter, in its own filesystem hierarchy (try os.listdir('/')). For example, you would run an untrusted diff --git a/source/features.txt b/source/features.txt --- a/source/features.txt +++ b/source/features.txt @@ -113,6 +113,7 @@ cd pypy/sandbox pypy_interact.py path/to/pypy-sandbox + # don't confuse it with pypy/goal/pyinteractive.py! You get a fully sandboxed interpreter, in its own filesystem hierarchy (try ``os.listdir('/')``). For example, you would run an untrusted From noreply at buildbot.pypy.org Mon Apr 27 10:22:15 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 27 Apr 2015 10:22:15 +0200 (CEST) Subject: [pypy-commit] pypy.org extradoc: Expand the warning. Message-ID: <20150427082215.E7FBB1C0EC0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r599:270cd2221470 Date: 2015-04-27 10:22 +0200 http://bitbucket.org/pypy/pypy.org/changeset/270cd2221470/ Log: Expand the warning. diff --git a/features.html b/features.html --- a/features.html +++ b/features.html @@ -136,8 +136,21 @@ try to limit language features considered “unsafe”. Instead we replace all calls to external libraries (C or platform) with a stub that communicates with an external process handling the policy.

Please be aware that it is a prototype only. It needs work to become -more complete, and you are welcome to help.

+ +++ + + + + +

Please be aware that it is a prototype only. It needs work to become +more complete, and you are welcome to help. In particular, almost none +of the extension modules work (not even time ), and pypy_interact +is merely a demo. Also, a more complete system would include a way +to do the same as pypy_interact from other languages than Python, +to embed a sandboxed interpreter inside programs written in other +languages.

To run the sandboxed process, you need to get the full sources and build pypy-sandbox from it (see Building from source). These instructions give you a pypy-c that you should rename to diff --git a/source/features.txt b/source/features.txt --- a/source/features.txt +++ b/source/features.txt @@ -103,8 +103,15 @@ calls to external libraries (C or platform) with a stub that communicates with an external process handling the policy. -Please be aware that it is a **prototype** only. It needs work to become -more complete, and you are welcome to help. ++-----------------------------------------------------------------------------+ +| **Please be aware that it is a prototype only.** *It needs work to become | +| more complete, and you are welcome to help. In particular, almost none | +| of the extension modules work (not even* ``time`` *), and* ``pypy_interact``| +| *is merely a demo. Also, a more complete system would include a way | +| to do the same as* ``pypy_interact`` *from other languages than Python, | +| to embed a sandboxed interpreter inside programs written in other | +| languages.* | ++-----------------------------------------------------------------------------+ To run the sandboxed process, you need to get the full sources and build ``pypy-sandbox`` from it (see `Building from source`_). These From noreply at buildbot.pypy.org Mon Apr 27 16:18:18 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 27 Apr 2015 16:18:18 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Move the guarantee of uniqueness of types to the core _cffi_backend.c. Message-ID: <20150427141818.2CB2F1C1427@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1855:04e073297436 Date: 2015-04-27 15:33 +0200 http://bitbucket.org/cffi/cffi/changeset/04e073297436/ Log: Move the guarantee of uniqueness of types to the core _cffi_backend.c. Gets rid of USES_LOCAL which is not really compatible with ffi.include(). diff --git a/_cffi1/ffi_obj.c b/_cffi1/ffi_obj.c --- a/_cffi1/ffi_obj.c +++ b/_cffi1/ffi_obj.c @@ -400,7 +400,6 @@ } z = new_pointer_type(ct); - z = get_unique_type(self->types_builder, z); if (z == NULL) return NULL; diff --git a/_cffi1/realize_c_type.c b/_cffi1/realize_c_type.c --- a/_cffi1/realize_c_type.c +++ b/_cffi1/realize_c_type.c @@ -7,7 +7,6 @@ static PyObject *all_primitives[_CFFI__NUM_PRIM]; -static PyObject *global_types_dict; static CTypeDescrObject *g_ct_voidp, *g_ct_chararray; static PyObject *build_primitive_type(int num); /* forward */ @@ -16,14 +15,6 @@ (all_primitives[num] != NULL ? all_primitives[num] \ : build_primitive_type(num)) -static int _add_to_global_types_dict(PyObject *ct) -{ - if (ct == NULL) - return -1; - return PyDict_SetItemString(global_types_dict, - ((CTypeDescrObject *)ct)->ct_name, ct); -} - static int init_global_types_dict(PyObject *ffi_type_dict) { int err; @@ -32,29 +23,25 @@ MemoryErrors during importing an extension module are kind of bad anyway */ - global_types_dict = PyDict_New(); - if (global_types_dict == NULL) - return -1; - ct_void = get_primitive_type(_CFFI_PRIM_VOID); // 'void' - if (_add_to_global_types_dict(ct_void) < 0) + if (ct_void == NULL) return -1; ct2 = new_pointer_type((CTypeDescrObject *)ct_void); // 'void *' - if (_add_to_global_types_dict(ct2) < 0) + if (ct2 == NULL) return -1; g_ct_voidp = (CTypeDescrObject *)ct2; ct_char = get_primitive_type(_CFFI_PRIM_CHAR); // 'char' - if (_add_to_global_types_dict(ct_char) < 0) + if (ct_char == NULL) return -1; ct2 = new_pointer_type((CTypeDescrObject *)ct_char); // 'char *' - if (_add_to_global_types_dict(ct2) < 0) + if (ct2 == NULL) return -1; ct2 = new_array_type((CTypeDescrObject *)ct2, -1); // 'char[]' - if (_add_to_global_types_dict(ct2) < 0) + if (ct2 == NULL) return -1; g_ct_chararray = (CTypeDescrObject *)ct2; @@ -117,66 +104,6 @@ return builder; } -static PyObject *get_unique_type(builder_c_t *builder, PyObject *x) -{ - /* Replace the CTypeDescrObject 'x' with a standardized one. - This either just returns x, or x is decrefed and a new reference - to the standard equivalent is returned. - - In this function, 'x' always contains a reference that must be - decrefed, and 'y' never does. - */ - CTypeDescrObject *ct = (CTypeDescrObject *)x; - if (ct == NULL) - return NULL; - - /* XXX maybe change the type of ct_name to be a real 'PyObject *'? */ - PyObject *name = PyString_FromString(ct->ct_name); - if (name == NULL) - goto no_memory; - - PyObject *y = PyDict_GetItem(builder->types_dict, name); - if (y != NULL) { - /* Already found the same ct_name in the dict. Return the old one. */ - Py_INCREF(y); - Py_DECREF(x); - x = y; - goto done; - } - - if (!(ct->ct_flags & CT_USES_LOCAL)) { - /* The type is not "local", i.e. does not make use of any struct, - union or enum. This means it should be shared across independent - ffi instances. Look it up and possibly add it to the global - types dict. - */ - y = PyDict_GetItem(global_types_dict, name); - if (y != NULL) { - Py_INCREF(y); - Py_DECREF(x); - x = y; - } - else { - /* Not found in the global dictionary. Put it there. */ - if (PyDict_SetItem(global_types_dict, name, x) < 0) - goto no_memory; - } - } - - /* Set x in the local dict. */ - if (PyDict_SetItem(builder->types_dict, name, x) < 0) - goto no_memory; - - done: - Py_DECREF(name); - return x; - - no_memory: - Py_XDECREF(name); - Py_DECREF(x); - return NULL; -} - static PyObject *build_primitive_type(int num) { /* XXX too many translations between here and new_primitive_type() */ @@ -365,7 +292,6 @@ return NULL; if (CTypeDescr_Check(y)) { x = new_pointer_type((CTypeDescrObject *)y); - x = get_unique_type(builder, x); } else { assert(PyTuple_Check(y)); /* from _CFFI_OP_FUNCTION */ @@ -383,12 +309,10 @@ if (y == NULL) return NULL; z = new_pointer_type((CTypeDescrObject *)y); - z = get_unique_type(builder, z); Py_DECREF(y); if (z == NULL) return NULL; x = new_array_type((CTypeDescrObject *)z, length); - x = get_unique_type(builder, x); Py_DECREF(z); break; @@ -573,7 +497,6 @@ z = new_function_type(fargs, (CTypeDescrObject *)y, ellipsis, FFI_DEFAULT_ABI); - z = get_unique_type(builder, z); Py_DECREF(fargs); Py_DECREF(y); if (z == NULL) diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -132,7 +132,6 @@ #define CT_WITH_VAR_ARRAY 1048576 #define CT_IS_UNSIZED_CHAR_A 2097152 #define CT_LAZY_FIELD_LIST 4194304 -#define CT_USES_LOCAL 8388608 #define CT_PRIMITIVE_ANY (CT_PRIMITIVE_SIGNED | \ CT_PRIMITIVE_UNSIGNED | \ CT_PRIMITIVE_CHAR | \ @@ -3381,6 +3380,57 @@ /************************************************************/ +static PyObject *unique_cache; + +static PyObject *get_unique_type(CTypeDescrObject *x, + const void *unique_key[], long keylength) +{ + /* Replace the CTypeDescrObject 'x' with a standardized one. + This either just returns x, or x is decrefed and a new reference + to the already-existing equivalent is returned. + + In this function, 'x' always contains a reference that must be + either decrefed or returned. + + Keys: + void ["void"] + primitive [&static_struct] + pointer [ctype] + array [ctype, length] + funcptr [ctresult, ellipsis+abi, num_args, ctargs...] + */ + long i; + PyObject *key, *y; + const void **pkey; + int err; + + key = PyString_FromStringAndSize(NULL, keylength * sizeof(void *)); + if (key == NULL) + goto error; + + pkey = (const void **)PyString_AS_STRING(key); + for (i = 0; i < keylength; i++) + pkey[i] = unique_key[i]; + + y = PyDict_GetItem(unique_cache, key); + if (y != NULL) { + Py_DECREF(key); + Py_INCREF(y); + Py_DECREF(x); + return y; + } + err = PyDict_SetItem(unique_cache, key, (PyObject *)x); + Py_DECREF(key); + if (err < 0) + goto error; + + return (PyObject *)x; + + error: + Py_DECREF(x); + return NULL; +} + static PyObject *new_primitive_type(const char *name) { #define ENUM_PRIMITIVE_TYPES \ @@ -3461,6 +3511,7 @@ { NULL } }; const struct descr_s *ptypes; + const void *unique_key[1]; int name_size; ffi_type *ffitype; @@ -3526,7 +3577,8 @@ td->ct_flags |= CT_PRIMITIVE_FITS_LONG; } td->ct_name_position = strlen(td->ct_name); - return (PyObject *)td; + unique_key[0] = ptypes; + return get_unique_type(td, unique_key, 1); bad_ffi_type: PyErr_Format(PyExc_NotImplementedError, @@ -3548,6 +3600,7 @@ { CTypeDescrObject *td; const char *extra; + const void *unique_key[1]; if (ctitem->ct_flags & CT_ARRAY) extra = "(*)"; /* obscure case: see test_array_add */ @@ -3559,7 +3612,7 @@ td->ct_size = sizeof(void *); td->ct_length = -1; - td->ct_flags = CT_POINTER | (ctitem->ct_flags & CT_USES_LOCAL); + td->ct_flags = CT_POINTER; if (ctitem->ct_flags & (CT_STRUCT|CT_UNION)) td->ct_flags |= CT_IS_PTR_TO_OWNED; if (ctitem->ct_flags & CT_VOID) @@ -3568,7 +3621,8 @@ ((ctitem->ct_flags & CT_PRIMITIVE_CHAR) && ctitem->ct_size == sizeof(char))) td->ct_flags |= CT_CAST_ANYTHING; /* 'void *' or 'char *' only */ - return (PyObject *)td; + unique_key[0] = ctitem; + return get_unique_type(td, unique_key, 1); } static PyObject *b_new_pointer_type(PyObject *self, PyObject *args) @@ -3611,6 +3665,7 @@ char extra_text[32]; Py_ssize_t arraysize; int flags = CT_ARRAY; + const void *unique_key[2]; if (!(ctptr->ct_flags & CT_POINTER)) { PyErr_SetString(PyExc_TypeError, "first arg must be a pointer ctype"); @@ -3648,13 +3703,16 @@ td->ct_stuff = (PyObject *)ctptr; td->ct_size = arraysize; td->ct_length = length; - td->ct_flags = flags | (ctptr->ct_flags & CT_USES_LOCAL); - return (PyObject *)td; + td->ct_flags = flags; + unique_key[0] = ctptr; + unique_key[1] = (void *)length; + return get_unique_type(td, unique_key, 2); } static PyObject *new_void_type(void) { int name_size = strlen("void") + 1; + const void *unique_key[1]; CTypeDescrObject *td = ctypedescr_new(name_size); if (td == NULL) return NULL; @@ -3663,7 +3721,8 @@ td->ct_size = -1; td->ct_flags = CT_VOID | CT_IS_OPAQUE; td->ct_name_position = strlen("void"); - return (PyObject *)td; + unique_key[0] = "void"; + return get_unique_type(td, unique_key, 1); } static PyObject *b_new_void_type(PyObject *self, PyObject *args) @@ -3680,7 +3739,7 @@ td->ct_size = -1; td->ct_length = -1; - td->ct_flags = flag | CT_IS_OPAQUE | CT_USES_LOCAL; + td->ct_flags = flag | CT_IS_OPAQUE; td->ct_extra = NULL; memcpy(td->ct_name, name, namelen + 1); td->ct_name_position = namelen; @@ -4406,8 +4465,6 @@ int ellipsis) { CTypeDescrObject *fct; - Py_ssize_t i, nargs; - int all_flags; fb->nb_bytes = 0; fb->bufferp = NULL; @@ -4429,16 +4486,9 @@ goto error; assert(fb->bufferp == fct->ct_name + fb->nb_bytes); - all_flags = fresult->ct_flags; - nargs = PyTuple_GET_SIZE(fargs); - for (i = 0; i < nargs; i++) { - CTypeDescrObject *farg = (CTypeDescrObject *)PyTuple_GET_ITEM(fargs, i); - all_flags |= farg->ct_flags; - } - fct->ct_extra = NULL; fct->ct_size = sizeof(void(*)(void)); - fct->ct_flags = CT_FUNCTIONPTR | (all_flags & CT_USES_LOCAL); + fct->ct_flags = CT_FUNCTIONPTR; return fct; error: @@ -4496,6 +4546,7 @@ CTypeDescrObject *fct; struct funcbuilder_s funcbuilder; Py_ssize_t i; + const void **unique_key; if ((fresult->ct_size < 0 && !(fresult->ct_flags & CT_VOID)) || (fresult->ct_flags & CT_ARRAY)) { @@ -4551,7 +4602,15 @@ Py_INCREF(o); PyTuple_SET_ITEM(fct->ct_stuff, 2 + i, o); } - return (PyObject *)fct; + + /* [ctresult, ellipsis+abi, num_args, ctargs...] */ + unique_key = alloca((3 + funcbuilder.nargs) * sizeof(void *)); + unique_key[0] = fresult; + unique_key[1] = (const void *)(Py_ssize_t)((fabi << 1) | !!ellipsis); + unique_key[2] = (const void *)(Py_ssize_t)(funcbuilder.nargs); + for (i=0; ict_stuff, 2 + i); + return get_unique_type(fct, unique_key, 3 + funcbuilder.nargs); error: Py_DECREF(fct); @@ -4890,7 +4949,7 @@ td->ct_size = basetd->ct_size; td->ct_length = basetd->ct_length; /* alignment */ td->ct_extra = basetd->ct_extra; /* ffi type */ - td->ct_flags = basetd->ct_flags | CT_IS_ENUM | CT_USES_LOCAL; + td->ct_flags = basetd->ct_flags | CT_IS_ENUM; td->ct_name_position = name_size - 1; return (PyObject *)td; @@ -5914,6 +5973,11 @@ if (m == NULL) INITERROR; + + unique_cache = PyDict_New(); + if (unique_cache == NULL) + INITERROR; + if (PyType_Ready(&dl_type) < 0) INITERROR; if (PyType_Ready(&CTypeDescr_Type) < 0) From noreply at buildbot.pypy.org Mon Apr 27 18:27:40 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 27 Apr 2015 18:27:40 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Check sizeof(global variables) when possible Message-ID: <20150427162740.29AEE1C0EF2@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1856:7d84833187ae Date: 2015-04-27 17:26 +0200 http://bitbucket.org/cffi/cffi/changeset/7d84833187ae/ Log: Check sizeof(global variables) when possible diff --git a/_cffi1/lib_obj.c b/_cffi1/lib_obj.c --- a/_cffi1/lib_obj.c +++ b/_cffi1/lib_obj.c @@ -141,8 +141,7 @@ PyErr_Format(PyExc_AttributeError, "lib '%.200s' has no function," " global variable or constant named '%.200s'", - PyText_AS_UTF8(lib->l_libname), - PyText_Check(name) ? PyText_AS_UTF8(name) : "?"); + PyText_AS_UTF8(lib->l_libname), s); return NULL; } @@ -198,12 +197,23 @@ _CFFI_GETARG(g->type_op)); if (ct == NULL) return NULL; - x = make_global_var(ct, g->address); + if (g->size != ct->ct_size && + g->size != (size_t)-1 && ct->ct_size != -1) { + PyErr_Format(FFIError, + "global variable '%.200s' should be %zd bytes " + "according to the cdef, but is actually %zd", + s, ct->ct_size, g->size); + x = NULL; + } + else { + x = make_global_var(ct, g->address); + } Py_DECREF(ct); break; default: - PyErr_SetString(PyExc_NotImplementedError, "in lib_build_attr"); + PyErr_Format(PyExc_NotImplementedError, "in lib_build_attr: op=%d", + (int)_CFFI_GETOP(g->type_op)); return NULL; } diff --git a/_cffi1/parse_c_type.h b/_cffi1/parse_c_type.h --- a/_cffi1/parse_c_type.h +++ b/_cffi1/parse_c_type.h @@ -63,6 +63,7 @@ struct _cffi_global_s { const char *name; void *address; + size_t size; // -1 if unknown _cffi_opcode_t type_op; }; diff --git a/_cffi1/recompiler.py b/_cffi1/recompiler.py --- a/_cffi1/recompiler.py +++ b/_cffi1/recompiler.py @@ -418,7 +418,8 @@ else: meth_kind = 'V' # 'METH_VARARGS' self._lsts["global"].append( - ' { "%s", _cffi_f_%s, _CFFI_OP(_CFFI_OP_CPYTHON_BLTN_%s, %d) },' + ' { "%s", _cffi_f_%s, (size_t)-1, ' + '_CFFI_OP(_CFFI_OP_CPYTHON_BLTN_%s, %d) },' % (name, name, meth_kind, type_index)) # ---------- @@ -631,7 +632,8 @@ type_index = self._typesdict[tp] type_op = '_CFFI_OP(_CFFI_OP_CONSTANT, %d)' % type_index self._lsts["global"].append( - ' { "%s", _cffi_const_%s, %s },' % (name, name, type_op)) + ' { "%s", _cffi_const_%s, (size_t)-1, %s },' % + (name, name, type_op)) # ---------- # enums @@ -648,8 +650,8 @@ type_op = '_CFFI_OP(_CFFI_OP_ENUM, -1)' for enumerator in tp.enumerators: self._lsts["global"].append( - ' { "%s", _cffi_const_%s, %s },' % (enumerator, enumerator, - type_op)) + ' { "%s", _cffi_const_%s, (size_t)-1, %s },' % + (enumerator, enumerator, type_op)) # if cname is not None and '$' not in cname: size = "sizeof(%s)" % cname @@ -679,7 +681,8 @@ def _generate_cpy_macro_ctx(self, tp, name): self._lsts["global"].append( - ' { "%s", _cffi_const_%s, _CFFI_OP(_CFFI_OP_CONSTANT_INT, 0) },' % + ' { "%s", _cffi_const_%s, (size_t)-1,' + ' _CFFI_OP(_CFFI_OP_CONSTANT_INT, 0) },' % (name, name)) # ---------- @@ -700,9 +703,13 @@ def _generate_cpy_variable_ctx(self, tp, name): tp = self._global_type(tp, name) type_index = self._typesdict[tp] + if tp.sizeof_enabled(): + size = "sizeof(%s)" % (name,) + else: + size = "(size_t)-1" self._lsts["global"].append( - ' { "%s", &%s, _CFFI_OP(_CFFI_OP_GLOBAL_VAR, %d)},' - % (name, name, type_index)) + ' { "%s", &%s, %s, _CFFI_OP(_CFFI_OP_GLOBAL_VAR, %d)},' + % (name, name, size, type_index)) # ---------- # emitting the opcodes for individual types diff --git a/_cffi1/test_recompiler.py b/_cffi1/test_recompiler.py --- a/_cffi1/test_recompiler.py +++ b/_cffi1/test_recompiler.py @@ -420,3 +420,23 @@ assert test_module_name_in_package.mymod.lib.foo(10) == 42 finally: sys.path[:] = old_sys_path + +def test_bad_size_of_global_1(): + ffi = FFI() + ffi.cdef("short glob;") + lib = verify(ffi, "test_bad_size_of_global_1", "long glob;") + py.test.raises(ffi.error, "lib.glob") + +def test_bad_size_of_global_2(): + ffi = FFI() + ffi.cdef("int glob[10];") + lib = verify(ffi, "test_bad_size_of_global_2", "int glob[9];") + e = py.test.raises(ffi.error, "lib.glob") + assert str(e.value) == ("global variable 'glob' should be 40 bytes " + "according to the cdef, but is actually 36") + +def test_unspecified_size_of_global(): + ffi = FFI() + ffi.cdef("int glob[];") + lib = verify(ffi, "test_unspecified_size_of_global", "int glob[10];") + lib.glob # does not crash diff --git a/cffi/model.py b/cffi/model.py --- a/cffi/model.py +++ b/cffi/model.py @@ -31,6 +31,9 @@ def has_c_name(self): return '$' not in self._get_c_name() + + def sizeof_enabled(self): + return False def get_cached_btype(self, ffi, finishlist, can_delay=False): try: @@ -121,6 +124,9 @@ def is_float_type(self): return self.ALL_PRIMITIVE_TYPES[self.name] == 'f' + def sizeof_enabled(self): + return True + def build_backend_type(self, ffi, finishlist): return global_cache(self, ffi, 'new_primitive_type', self.name) @@ -161,6 +167,9 @@ class FunctionPtrType(BaseFunctionType): _base_pattern = '(*&)(%s)' + def sizeof_enabled(self): + return True + def build_backend_type(self, ffi, finishlist): result = self.result.get_cached_btype(ffi, finishlist) args = [] @@ -186,6 +195,9 @@ extra = self._base_pattern self.c_name_with_marker = totype.c_name_with_marker.replace('&', extra) + def sizeof_enabled(self): + return True + def build_backend_type(self, ffi, finishlist): BItem = self.totype.get_cached_btype(ffi, finishlist, can_delay=True) return global_cache(self, ffi, 'new_pointer_type', BItem) @@ -226,6 +238,9 @@ self.c_name_with_marker = ( self.item.c_name_with_marker.replace('&', brackets)) + def sizeof_enabled(self): + return self.item.sizeof_enabled() and self.length is not None + def resolve_length(self, newlength): return ArrayType(self.item, newlength) @@ -379,6 +394,9 @@ from . import ffiplatform raise ffiplatform.VerificationMissing(self._get_c_name()) + def sizeof_enabled(self): + return self.fldtypes is not None + def build_backend_type(self, ffi, finishlist): self.check_not_partial() finishlist.append(self) @@ -407,6 +425,9 @@ self.baseinttype = baseinttype self.build_c_name_with_marker() + def sizeof_enabled(self): + return True # not strictly true, but external enums are obscure + def force_the_name(self, forcename): StructOrUnionOrEnum.force_the_name(self, forcename) if self.forcename is None: From noreply at buildbot.pypy.org Mon Apr 27 18:34:05 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 27 Apr 2015 18:34:05 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Move the "size" field to the end of the structure and use 0 to mean "unknown" Message-ID: <20150427163405.99B951C116B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1857:107bf9987254 Date: 2015-04-27 18:34 +0200 http://bitbucket.org/cffi/cffi/changeset/107bf9987254/ Log: Move the "size" field to the end of the structure and use 0 to mean "unknown" diff --git a/_cffi1/lib_obj.c b/_cffi1/lib_obj.c --- a/_cffi1/lib_obj.c +++ b/_cffi1/lib_obj.c @@ -197,8 +197,7 @@ _CFFI_GETARG(g->type_op)); if (ct == NULL) return NULL; - if (g->size != ct->ct_size && - g->size != (size_t)-1 && ct->ct_size != -1) { + if (g->size != ct->ct_size && g->size != 0 && ct->ct_size > 0) { PyErr_Format(FFIError, "global variable '%.200s' should be %zd bytes " "according to the cdef, but is actually %zd", diff --git a/_cffi1/parse_c_type.h b/_cffi1/parse_c_type.h --- a/_cffi1/parse_c_type.h +++ b/_cffi1/parse_c_type.h @@ -63,8 +63,8 @@ struct _cffi_global_s { const char *name; void *address; - size_t size; // -1 if unknown _cffi_opcode_t type_op; + size_t size; // 0 if unknown }; struct _cffi_struct_union_s { diff --git a/_cffi1/recompiler.py b/_cffi1/recompiler.py --- a/_cffi1/recompiler.py +++ b/_cffi1/recompiler.py @@ -418,8 +418,7 @@ else: meth_kind = 'V' # 'METH_VARARGS' self._lsts["global"].append( - ' { "%s", _cffi_f_%s, (size_t)-1, ' - '_CFFI_OP(_CFFI_OP_CPYTHON_BLTN_%s, %d) },' + ' { "%s", _cffi_f_%s, _CFFI_OP(_CFFI_OP_CPYTHON_BLTN_%s, %d), 0 },' % (name, name, meth_kind, type_index)) # ---------- @@ -632,8 +631,7 @@ type_index = self._typesdict[tp] type_op = '_CFFI_OP(_CFFI_OP_CONSTANT, %d)' % type_index self._lsts["global"].append( - ' { "%s", _cffi_const_%s, (size_t)-1, %s },' % - (name, name, type_op)) + ' { "%s", _cffi_const_%s, %s, 0 },' % (name, name, type_op)) # ---------- # enums @@ -650,7 +648,7 @@ type_op = '_CFFI_OP(_CFFI_OP_ENUM, -1)' for enumerator in tp.enumerators: self._lsts["global"].append( - ' { "%s", _cffi_const_%s, (size_t)-1, %s },' % + ' { "%s", _cffi_const_%s, %s, 0 },' % (enumerator, enumerator, type_op)) # if cname is not None and '$' not in cname: @@ -681,9 +679,8 @@ def _generate_cpy_macro_ctx(self, tp, name): self._lsts["global"].append( - ' { "%s", _cffi_const_%s, (size_t)-1,' - ' _CFFI_OP(_CFFI_OP_CONSTANT_INT, 0) },' % - (name, name)) + ' { "%s", _cffi_const_%s,' + ' _CFFI_OP(_CFFI_OP_CONSTANT_INT, 0), 0 },' % (name, name)) # ---------- # global variables @@ -706,10 +703,10 @@ if tp.sizeof_enabled(): size = "sizeof(%s)" % (name,) else: - size = "(size_t)-1" + size = "0" self._lsts["global"].append( - ' { "%s", &%s, %s, _CFFI_OP(_CFFI_OP_GLOBAL_VAR, %d)},' - % (name, name, size, type_index)) + ' { "%s", &%s, _CFFI_OP(_CFFI_OP_GLOBAL_VAR, %d), %s },' + % (name, name, type_index, size)) # ---------- # emitting the opcodes for individual types From noreply at buildbot.pypy.org Mon Apr 27 19:04:40 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 27 Apr 2015 19:04:40 +0200 (CEST) Subject: [pypy-commit] cffi default: Test and fix: the new types like "int_least8_t" were actually not Message-ID: <20150427170440.5F0881C115A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1858:a7b42ee0db8e Date: 2015-04-27 19:04 +0200 http://bitbucket.org/cffi/cffi/changeset/a7b42ee0db8e/ Log: Test and fix: the new types like "int_least8_t" were actually not available because model.py didn't list them diff --git a/cffi/model.py b/cffi/model.py --- a/cffi/model.py +++ b/cffi/model.py @@ -102,8 +102,26 @@ 'uint32_t': 'i', 'int64_t': 'i', 'uint64_t': 'i', + 'int_least8_t': 'i', + 'uint_least8_t': 'i', + 'int_least16_t': 'i', + 'uint_least16_t': 'i', + 'int_least32_t': 'i', + 'uint_least32_t': 'i', + 'int_least64_t': 'i', + 'uint_least64_t': 'i', + 'int_fast8_t': 'i', + 'uint_fast8_t': 'i', + 'int_fast16_t': 'i', + 'uint_fast16_t': 'i', + 'int_fast32_t': 'i', + 'uint_fast32_t': 'i', + 'int_fast64_t': 'i', + 'uint_fast64_t': 'i', 'intptr_t': 'i', 'uintptr_t': 'i', + 'intmax_t': 'i', + 'uintmax_t': 'i', 'ptrdiff_t': 'i', 'size_t': 'i', 'ssize_t': 'i', diff --git a/testing/backend_tests.py b/testing/backend_tests.py --- a/testing/backend_tests.py +++ b/testing/backend_tests.py @@ -1703,5 +1703,3 @@ assert lib.DOT_HEX == 0x100 assert lib.DOT_HEX2 == 0x10 assert lib.DOT_UL == 1000 - - diff --git a/testing/test_ffi_backend.py b/testing/test_ffi_backend.py --- a/testing/test_ffi_backend.py +++ b/testing/test_ffi_backend.py @@ -222,3 +222,57 @@ assert ffi.typeof(c) is ffi.typeof("char[]") ffi.cast("unsigned short *", c)[1] += 500 assert list(a) == [10000, 20500, 30000] + + def test_all_primitives(self): + ffi = FFI() + for name in [ + "char", + "short", + "int", + "long", + "long long", + "signed char", + "unsigned char", + "unsigned short", + "unsigned int", + "unsigned long", + "unsigned long long", + "float", + "double", + "long double", + "wchar_t", + "_Bool", + "int8_t", + "uint8_t", + "int16_t", + "uint16_t", + "int32_t", + "uint32_t", + "int64_t", + "uint64_t", + "int_least8_t", + "uint_least8_t", + "int_least16_t", + "uint_least16_t", + "int_least32_t", + "uint_least32_t", + "int_least64_t", + "uint_least64_t", + "int_fast8_t", + "uint_fast8_t", + "int_fast16_t", + "uint_fast16_t", + "int_fast32_t", + "uint_fast32_t", + "int_fast64_t", + "uint_fast64_t", + "intptr_t", + "uintptr_t", + "intmax_t", + "uintmax_t", + "ptrdiff_t", + "size_t", + "ssize_t", + ]: + x = ffi.sizeof(name) + assert 1 <= x <= 16 From noreply at buildbot.pypy.org Mon Apr 27 19:09:47 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 27 Apr 2015 19:09:47 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: hg merge default Message-ID: <20150427170947.E4A541C04A4@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1859:283af058e34b Date: 2015-04-27 19:10 +0200 http://bitbucket.org/cffi/cffi/changeset/283af058e34b/ Log: hg merge default diff --git a/cffi/model.py b/cffi/model.py --- a/cffi/model.py +++ b/cffi/model.py @@ -105,8 +105,26 @@ 'uint32_t': 'i', 'int64_t': 'i', 'uint64_t': 'i', + 'int_least8_t': 'i', + 'uint_least8_t': 'i', + 'int_least16_t': 'i', + 'uint_least16_t': 'i', + 'int_least32_t': 'i', + 'uint_least32_t': 'i', + 'int_least64_t': 'i', + 'uint_least64_t': 'i', + 'int_fast8_t': 'i', + 'uint_fast8_t': 'i', + 'int_fast16_t': 'i', + 'uint_fast16_t': 'i', + 'int_fast32_t': 'i', + 'uint_fast32_t': 'i', + 'int_fast64_t': 'i', + 'uint_fast64_t': 'i', 'intptr_t': 'i', 'uintptr_t': 'i', + 'intmax_t': 'i', + 'uintmax_t': 'i', 'ptrdiff_t': 'i', 'size_t': 'i', 'ssize_t': 'i', diff --git a/testing/backend_tests.py b/testing/backend_tests.py --- a/testing/backend_tests.py +++ b/testing/backend_tests.py @@ -1703,5 +1703,3 @@ assert lib.DOT_HEX == 0x100 assert lib.DOT_HEX2 == 0x10 assert lib.DOT_UL == 1000 - - diff --git a/testing/test_ffi_backend.py b/testing/test_ffi_backend.py --- a/testing/test_ffi_backend.py +++ b/testing/test_ffi_backend.py @@ -222,3 +222,57 @@ assert ffi.typeof(c) is ffi.typeof("char[]") ffi.cast("unsigned short *", c)[1] += 500 assert list(a) == [10000, 20500, 30000] + + def test_all_primitives(self): + ffi = FFI() + for name in [ + "char", + "short", + "int", + "long", + "long long", + "signed char", + "unsigned char", + "unsigned short", + "unsigned int", + "unsigned long", + "unsigned long long", + "float", + "double", + "long double", + "wchar_t", + "_Bool", + "int8_t", + "uint8_t", + "int16_t", + "uint16_t", + "int32_t", + "uint32_t", + "int64_t", + "uint64_t", + "int_least8_t", + "uint_least8_t", + "int_least16_t", + "uint_least16_t", + "int_least32_t", + "uint_least32_t", + "int_least64_t", + "uint_least64_t", + "int_fast8_t", + "uint_fast8_t", + "int_fast16_t", + "uint_fast16_t", + "int_fast32_t", + "uint_fast32_t", + "int_fast64_t", + "uint_fast64_t", + "intptr_t", + "uintptr_t", + "intmax_t", + "uintmax_t", + "ptrdiff_t", + "size_t", + "ssize_t", + ]: + x = ffi.sizeof(name) + assert 1 <= x <= 16 From noreply at buildbot.pypy.org Mon Apr 27 19:13:07 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 27 Apr 2015 19:13:07 +0200 (CEST) Subject: [pypy-commit] cffi default: Mention the bug Message-ID: <20150427171307.1706E1C04A4@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1860:ccbd4be0624d Date: 2015-04-27 19:13 +0200 http://bitbucket.org/cffi/cffi/changeset/ccbd4be0624d/ Log: Mention the bug diff --git a/doc/source/index.rst b/doc/source/index.rst --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -437,9 +437,9 @@ types ``TBYTE TCHAR LPCTSTR PCTSTR LPTSTR PTSTR PTBYTE PTCHAR`` are no longer automatically defined; see ``ffi.set_unicode()`` below. -* *New in version 0.9:* the other standard integer types from stdint.h, +* *New in version 0.9.3:* the other standard integer types from stdint.h, as long as they map to integers of 1, 2, 4 or 8 bytes. Larger integers - are not supported. + are not supported. (Actually added in version 0.9 but this was buggy.) .. _`common Windows types`: http://msdn.microsoft.com/en-us/library/windows/desktop/aa383751%28v=vs.85%29.aspx From noreply at buildbot.pypy.org Mon Apr 27 21:31:36 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 27 Apr 2015 21:31:36 +0200 (CEST) Subject: [pypy-commit] pypy default: Add a @signature here (seen by translating Message-ID: <20150427193136.A0E821C114E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76946:76b0dfb05efb Date: 2015-04-27 21:31 +0200 http://bitbucket.org/pypy/pypy/changeset/76b0dfb05efb/ Log: Add a @signature here (seen by translating https://github.com/cgswords/laurens/blob/master/laurens/stg.py at baa4326) diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py --- a/rpython/rtyper/lltypesystem/rordereddict.py +++ b/rpython/rtyper/lltypesystem/rordereddict.py @@ -3,7 +3,8 @@ from rpython.flowspace.model import Constant from rpython.rtyper.rdict import AbstractDictRepr, AbstractDictIteratorRepr from rpython.rtyper.lltypesystem import lltype, llmemory, rffi -from rpython.rlib import objectmodel, jit, rgc +from rpython.rlib import objectmodel, jit, rgc, types +from rpython.rlib.signature import signature from rpython.rlib.objectmodel import specialize, likely from rpython.rlib.debug import ll_assert from rpython.rlib.rarithmetic import r_uint, intmask @@ -515,9 +516,11 @@ dummy = ENTRIES.dummy_obj.ll_dummy_value entries[i].value = dummy + at signature(types.any(), types.int(), returns=types.any()) def ll_hash_from_cache(entries, i): return entries[i].f_hash + at signature(types.any(), types.int(), returns=types.any()) def ll_hash_recomputed(entries, i): ENTRIES = lltype.typeOf(entries).TO return ENTRIES.fasthashfn(entries[i].key) From noreply at buildbot.pypy.org Mon Apr 27 23:48:54 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 27 Apr 2015 23:48:54 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Move this as a static global Message-ID: <20150427214854.50B7E1C0EC0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1863:2f227a0cb551 Date: 2015-04-27 21:56 +0200 http://bitbucket.org/cffi/cffi/changeset/2f227a0cb551/ Log: Move this as a static global diff --git a/_cffi1/ffi_obj.c b/_cffi1/ffi_obj.c --- a/_cffi1/ffi_obj.c +++ b/_cffi1/ffi_obj.c @@ -24,12 +24,13 @@ struct _cffi_parse_info_s info; int ctx_is_static; builder_c_t *types_builder; - _cffi_opcode_t internal_output[FFI_COMPLEXITY_OUTPUT]; }; static FFIObject *ffi_internal_new(PyTypeObject *ffitype, const struct _cffi_type_context_s *static_ctx) { + static _cffi_opcode_t internal_output[FFI_COMPLEXITY_OUTPUT]; + FFIObject *ffi; if (static_ctx != NULL) { ffi = (FFIObject *)PyObject_GC_New(FFIObject, ffitype); @@ -49,7 +50,7 @@ } ffi->gc_wrefs = NULL; ffi->info.ctx = &ffi->types_builder->ctx; - ffi->info.output = ffi->internal_output; + ffi->info.output = internal_output; ffi->info.output_size = FFI_COMPLEXITY_OUTPUT; ffi->ctx_is_static = (static_ctx != NULL); #if 0 From noreply at buildbot.pypy.org Mon Apr 27 23:48:55 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 27 Apr 2015 23:48:55 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Basic idea: we ffi.include(baseffi), and baseffi must also be turned into Message-ID: <20150427214855.518251C0EC0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1864:bebb7929ea1f Date: 2015-04-27 21:58 +0200 http://bitbucket.org/cffi/cffi/changeset/bebb7929ea1f/ Log: Basic idea: we ffi.include(baseffi), and baseffi must also be turned into C code when ffi is. Then the name of the C module from baseffi ends up written in the global data for ffi. diff --git a/_cffi1/parse_c_type.h b/_cffi1/parse_c_type.h --- a/_cffi1/parse_c_type.h +++ b/_cffi1/parse_c_type.h @@ -113,6 +113,7 @@ int num_struct_unions; int num_enums; int num_typenames; + const char * const *includes; }; struct _cffi_parse_info_s { diff --git a/_cffi1/recompiler.py b/_cffi1/recompiler.py --- a/_cffi1/recompiler.py +++ b/_cffi1/recompiler.py @@ -192,6 +192,19 @@ assert lst[i].startswith(' { "%s"' % tp.name) assert len(lst) == len(self._enums) # + # the declaration of '_cffi_includes' + if self.ffi._included_ffis: + prnt('static const char * const _cffi_includes[] = {') + for ffi_to_include in self.ffi._included_ffis: + if not hasattr(ffi_to_include, '_recompiler_module_name'): + raise ffiplatform.VerificationError( + "this ffi includes %r, but the latter has not been " + "turned into a C module" % (ffi_to_include,)) + prnt(' "%s",' % (ffi_to_include._recompiler_module_name,)) + prnt(' NULL') + prnt('};') + prnt() + # # the declaration of '_cffi_type_context' prnt('static const struct _cffi_type_context_s _cffi_type_context = {') prnt(' _cffi_types,') @@ -203,6 +216,10 @@ for step_name in ALL_STEPS: if step_name != "field": prnt(' %d, /* num_%ss */' % (nums[step_name], step_name)) + if self.ffi._included_ffis: + prnt(' _cffi_includes,') + else: + prnt(' NULL, /* no includes */') prnt('};') prnt() # @@ -216,6 +233,7 @@ prnt(' _cffi_init_module("%s", &_cffi_type_context);' % ( self.module_name,)) prnt('}') + self.ffi._recompiler_module_name = self.module_name # ---------- diff --git a/_cffi1/test_recompiler.py b/_cffi1/test_recompiler.py --- a/_cffi1/test_recompiler.py +++ b/_cffi1/test_recompiler.py @@ -456,7 +456,9 @@ lib.glob # does not crash def test_include_1(): - ffi1 = FFI(); ffi1.cdef("typedef double foo_t;") + ffi1 = FFI() + ffi1.cdef("typedef double foo_t;") + verify(ffi1, "test_include_1_parent", "typedef double foo_t;") ffi = FFI() ffi.include(ffi1) ffi.cdef("foo_t ff1(foo_t);") @@ -464,7 +466,9 @@ assert lib.ff1(0) == 42.5 def test_include_2(): - ffi1 = FFI(); ffi1.cdef("struct foo_s { int x, y; };") + ffi1 = FFI() + ffi1.cdef("struct foo_s { int x, y; };") + verify(ffi1, "test_include_2_parent", "struct foo_s { int x, y; };") ffi = FFI() ffi.include(ffi1) ffi.cdef("struct foo_s *ff2(struct foo_s *);") diff --git a/cffi/api.py b/cffi/api.py --- a/cffi/api.py +++ b/cffi/api.py @@ -70,6 +70,7 @@ self._function_caches = [] self._libraries = [] self._cdefsources = [] + self._included_ffis = [] self._windows_unicode = None if hasattr(backend, 'set_ffi'): backend.set_ffi(self) @@ -424,6 +425,7 @@ self._cdefsources.append('[') self._cdefsources.extend(ffi_to_include._cdefsources) self._cdefsources.append(']') + self._included_ffis.append(ffi_to_include) def new_handle(self, x): return self._backend.newp_handle(self.BVoidP, x) From noreply at buildbot.pypy.org Mon Apr 27 23:48:53 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 27 Apr 2015 23:48:53 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Comment out ffi.__set_types(), never used for now Message-ID: <20150427214853.579EA1C0EC0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1862:987e9d3d69c4 Date: 2015-04-27 18:53 +0200 http://bitbucket.org/cffi/cffi/changeset/987e9d3d69c4/ Log: Comment out ffi.__set_types(), never used for now diff --git a/_cffi1/ffi_obj.c b/_cffi1/ffi_obj.c --- a/_cffi1/ffi_obj.c +++ b/_cffi1/ffi_obj.c @@ -24,7 +24,6 @@ struct _cffi_parse_info_s info; int ctx_is_static; builder_c_t *types_builder; - PyObject *dynamic_types; _cffi_opcode_t internal_output[FFI_COMPLEXITY_OUTPUT]; }; @@ -53,7 +52,9 @@ ffi->info.output = ffi->internal_output; ffi->info.output_size = FFI_COMPLEXITY_OUTPUT; ffi->ctx_is_static = (static_ctx != NULL); +#if 0 ffi->dynamic_types = NULL; +#endif return ffi; } @@ -61,7 +62,9 @@ { PyObject_GC_UnTrack(ffi); Py_XDECREF(ffi->gc_wrefs); +#if 0 Py_XDECREF(ffi->dynamic_types); +#endif if (!ffi->ctx_is_static) free_builder_c(ffi->types_builder); @@ -640,6 +643,7 @@ return 0; } +#if 0 static PyObject *ffi__set_types(FFIObject *self, PyObject *args) { PyObject *lst1, *lst2; @@ -726,9 +730,9 @@ Py_INCREF(Py_None); return Py_None; } +#endif static PyMethodDef ffi_methods[] = { - {"__set_types",(PyCFunction)ffi__set_types, METH_VARARGS}, {"addressof", (PyCFunction)ffi_addressof, METH_VARARGS, ffi_addressof_doc}, {"alignof", (PyCFunction)ffi_alignof, METH_O, ffi_alignof_doc}, {"buffer", (PyCFunction)ffi_buffer, METH_VARARGS, ffi_buffer_doc}, diff --git a/_cffi1/realize_c_type.c b/_cffi1/realize_c_type.c --- a/_cffi1/realize_c_type.c +++ b/_cffi1/realize_c_type.c @@ -2,7 +2,6 @@ typedef struct { struct _cffi_type_context_s ctx; /* inlined substructure */ PyObject *types_dict; - int num_types_imported; } builder_c_t; @@ -56,12 +55,14 @@ static void cleanup_builder_c(builder_c_t *builder) { int i; +#if 0 for (i = builder->num_types_imported; (--i) >= 0; ) { _cffi_opcode_t x = builder->ctx.types[i]; if ((((uintptr_t)x) & 1) == 0) { Py_XDECREF((PyObject *)x); } } +#endif const void *mem[] = {builder->ctx.types, builder->ctx.globals, @@ -100,7 +101,9 @@ memset(&builder->ctx, 0, sizeof(builder->ctx)); builder->types_dict = ldict; +#if 0 builder->num_types_imported = 0; +#endif return builder; } From noreply at buildbot.pypy.org Mon Apr 27 23:48:52 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 27 Apr 2015 23:48:52 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: start on ffi.include Message-ID: <20150427214852.4151B1C0EC0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1861:fa462bbfa883 Date: 2015-04-27 18:27 +0200 http://bitbucket.org/cffi/cffi/changeset/fa462bbfa883/ Log: start on ffi.include diff --git a/_cffi1/parse_c_type.h b/_cffi1/parse_c_type.h --- a/_cffi1/parse_c_type.h +++ b/_cffi1/parse_c_type.h @@ -80,6 +80,7 @@ #define _CFFI_F_CHECK_FIELDS 0x02 // complain if fields are not in the // "standard layout" or if some are missing #define _CFFI_F_PACKED 0x04 // for CHECK_FIELDS, assume a packed struct +#define _CFFI_F_EXTERNAL 0x08 // in some other ffi.include() struct _cffi_field_s { const char *name; diff --git a/_cffi1/recompiler.py b/_cffi1/recompiler.py --- a/_cffi1/recompiler.py +++ b/_cffi1/recompiler.py @@ -8,9 +8,6 @@ def __init__(self, ffi, module_name): self.ffi = ffi self.module_name = module_name - # - if ']' in self.ffi._cdefsources: - raise NotImplementedError("ffi.include()") def collect_type_table(self): self._typesdict = {} @@ -80,7 +77,8 @@ if isinstance(tp, model.FunctionPtrType): self._do_collect_type(tp.as_raw_function()) elif isinstance(tp, model.StructOrUnion): - if tp.fldtypes is not None: + if tp.fldtypes is not None and ( + tp not in self.ffi._parser._included_declarations): for name1, tp1, _ in tp.enumfields(): self._do_collect_type(self._field_type(tp, name1, tp1)) else: @@ -464,19 +462,24 @@ def _struct_ctx(self, tp, cname, approxname): type_index = self._typesdict[tp] + reason_for_not_expanding = None flags = [] if isinstance(tp, model.UnionType): flags.append("_CFFI_F_UNION") - if tp.fldtypes is None: - pass # opaque - elif tp.partial or tp.has_anonymous_struct_fields(): - pass # the field layout is obtained silently from the C compiler + if tp not in self.ffi._parser._included_declarations: + if tp.fldtypes is None: + reason_for_not_expanding = "opaque" + elif tp.partial or tp.has_anonymous_struct_fields(): + pass # field layout obtained silently from the C compiler + else: + flags.append("_CFFI_F_CHECK_FIELDS") + if tp.packed: + flags.append("_CFFI_F_PACKED") else: - flags.append("_CFFI_F_CHECK_FIELDS") - if tp.packed: - flags.append("_CFFI_F_PACKED") + flags.append("_CFFI_F_EXTERNAL") + reason_for_not_expanding = "external" flags = '|'.join(flags) or '0' - if tp.fldtypes is not None: + if reason_for_not_expanding is None: c_field = [approxname] enumfields = list(tp.enumfields()) for fldname, fldtype, fbitsize in enumfields: @@ -515,7 +518,8 @@ ' _cffi_FIELDS_FOR_%s, %d },' % (approxname, len(enumfields),)) else: - size_align = ' (size_t)-1, -1, -1, 0 /* opaque */ },' + size_align = ' (size_t)-1, -1, -1, 0 /* %s */ },' % ( + reason_for_not_expanding,) self._lsts["struct_union"].append( ' { "%s", %d, %s,' % (tp.name, type_index, flags) + size_align) self._seen_struct_unions.add(tp) diff --git a/_cffi1/test_recompiler.py b/_cffi1/test_recompiler.py --- a/_cffi1/test_recompiler.py +++ b/_cffi1/test_recompiler.py @@ -4,8 +4,12 @@ from _cffi1.udir import udir -def check_type_table(input, expected_output): +def check_type_table(input, expected_output, included=None): ffi = FFI() + if included: + ffi1 = FFI() + ffi1.cdef(included) + ffi.include(ffi1) ffi.cdef(input) recomp = recompiler.Recompiler(ffi, 'testmod') recomp.collect_type_table() @@ -92,6 +96,16 @@ check_type_table("enum foo_e { AA, BB, ... };", "(ENUM 0)") +def test_type_table_include_1(): + check_type_table("foo_t sin(foo_t);", + "(FUNCTION 1)(PRIMITIVE 14)(FUNCTION_END 0)", + included="typedef double foo_t;") + +def test_type_table_include_2(): + check_type_table("struct foo_s *sin(struct foo_s *);", + "(FUNCTION 1)(POINTER 3)(FUNCTION_END 0)(STRUCT_UNION 0)", + included="struct foo_s { int x, y; };") + def test_math_sin(): import math @@ -440,3 +454,25 @@ ffi.cdef("int glob[];") lib = verify(ffi, "test_unspecified_size_of_global", "int glob[10];") lib.glob # does not crash + +def test_include_1(): + ffi1 = FFI(); ffi1.cdef("typedef double foo_t;") + ffi = FFI() + ffi.include(ffi1) + ffi.cdef("foo_t ff1(foo_t);") + lib = verify(ffi, "test_include_1", "double ff1(double x) { return 42.5; }") + assert lib.ff1(0) == 42.5 + +def test_include_2(): + ffi1 = FFI(); ffi1.cdef("struct foo_s { int x, y; };") + ffi = FFI() + ffi.include(ffi1) + ffi.cdef("struct foo_s *ff2(struct foo_s *);") + lib = verify(ffi, "test_include_2", + "struct foo_s { int x, y; };\n" + "struct foo_s *ff2(struct foo_s *p) { p->y++; return p; }") + p = ffi.new("struct foo_s *") + p.y = 41 + q = lib.ff2(p) + assert q == p + assert p.y == 42 diff --git a/cffi/cparser.py b/cffi/cparser.py --- a/cffi/cparser.py +++ b/cffi/cparser.py @@ -95,6 +95,7 @@ def __init__(self): self._declarations = {} + self._included_declarations = set() self._anonymous_counter = 0 self._structnode2type = weakref.WeakKeyDictionary() self._override = False @@ -278,7 +279,7 @@ raise api.CDefError("unknown identifier '%s'" % (exprnode.name,)) return self._get_type(exprnode.type) - def _declare(self, name, obj): + def _declare(self, name, obj, included=False): if name in self._declarations: if self._declarations[name] is obj: return @@ -288,6 +289,8 @@ "try cdef(xx, override=True))" % (name,)) assert '__dotdotdot__' not in name.split() self._declarations[name] = obj + if included: + self._included_declarations.add(obj) def _get_type_pointer(self, type, const=False): if isinstance(type, model.RawFunctionType): @@ -601,7 +604,9 @@ def include(self, other): for name, tp in other._declarations.items(): kind = name.split(' ', 1)[0] - if kind in ('typedef', 'struct', 'union', 'enum'): + if kind in ('struct', 'union', 'enum'): + self._declare(name, tp, included=True) + elif kind == 'typedef': self._declare(name, tp) for k, v in other._int_constants.items(): self._add_constants(k, v) From noreply at buildbot.pypy.org Mon Apr 27 23:48:56 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 27 Apr 2015 23:48:56 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: look in ffi.included modules for global functions/variables/consts Message-ID: <20150427214856.4ED291C0EC0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1865:42497db90079 Date: 2015-04-27 22:56 +0200 http://bitbucket.org/cffi/cffi/changeset/42497db90079/ Log: look in ffi.included modules for global functions/variables/consts diff --git a/_cffi1/cffi1_module.c b/_cffi1/cffi1_module.c --- a/_cffi1/cffi1_module.c +++ b/_cffi1/cffi1_module.c @@ -45,6 +45,59 @@ return 0; } +static int make_included_tuples(const char *const *ctx_includes, + PyObject **included_ffis, + PyObject **included_libs) +{ + Py_ssize_t num = 0; + const char *const *p_include; + + if (ctx_includes == NULL) + return 0; + + for (p_include = ctx_includes; *p_include; p_include++) { + num++; + } + *included_ffis = PyTuple_New(num); + *included_libs = PyTuple_New(num); + if (*included_ffis == NULL || *included_libs == NULL) + goto error; + + num = 0; + for (p_include = ctx_includes; *p_include; p_include++) { + PyObject *included_ffi, *included_lib; + PyObject *m = PyImport_ImportModule(*p_include); + if (m == NULL) + goto error; + + included_ffi = PyObject_GetAttrString(m, "ffi"); + PyTuple_SET_ITEM(*included_ffis, num, included_ffi); + + included_lib = (included_ffi == NULL) ? NULL : + PyObject_GetAttrString(m, "lib"); + PyTuple_SET_ITEM(*included_libs, num, included_lib); + + Py_DECREF(m); + if (included_lib == NULL) + goto error; + + if (!FFIObject_Check(included_ffi) || + !LibObject_Check(included_lib)) { + PyErr_Format(PyExc_TypeError, + "expected FFI/Lib objects in %.200s.ffi/lib", + *p_include); + goto error; + } + num++; + } + return 0; + + error: + Py_XDECREF(*included_ffis); *included_ffis = NULL; + Py_XDECREF(*included_libs); *included_libs = NULL; + return -1; +} + static int _cffi_init_module(char *module_name, const struct _cffi_type_context_s *ctx) { @@ -61,5 +114,9 @@ if (lib == NULL || PyModule_AddObject(m, "lib", (PyObject *)lib) < 0) return -1; + if (make_included_tuples(ctx->includes, &ffi->included_ffis, + &lib->l_includes) < 0) + return -1; + return 0; } diff --git a/_cffi1/ffi_obj.c b/_cffi1/ffi_obj.c --- a/_cffi1/ffi_obj.c +++ b/_cffi1/ffi_obj.c @@ -18,12 +18,15 @@ #define FFI_COMPLEXITY_OUTPUT 1200 /* xxx should grow as needed */ +#define FFIObject_Check(op) PyObject_TypeCheck(op, &FFI_Type) + struct FFIObject_s { PyObject_HEAD PyObject *gc_wrefs; struct _cffi_parse_info_s info; int ctx_is_static; builder_c_t *types_builder; + PyObject *included_ffis; }; static FFIObject *ffi_internal_new(PyTypeObject *ffitype, @@ -53,6 +56,7 @@ ffi->info.output = internal_output; ffi->info.output_size = FFI_COMPLEXITY_OUTPUT; ffi->ctx_is_static = (static_ctx != NULL); + ffi->included_ffis = NULL; #if 0 ffi->dynamic_types = NULL; #endif @@ -70,6 +74,7 @@ if (!ffi->ctx_is_static) free_builder_c(ffi->types_builder); + Py_XDECREF(ffi->included_ffis); Py_TYPE(ffi)->tp_free((PyObject *)ffi); } @@ -77,6 +82,7 @@ { Py_VISIT(ffi->types_builder->types_dict); Py_VISIT(ffi->gc_wrefs); + Py_VISIT(ffi->included_ffis); return 0; } diff --git a/_cffi1/lib_obj.c b/_cffi1/lib_obj.c --- a/_cffi1/lib_obj.c +++ b/_cffi1/lib_obj.c @@ -23,6 +23,7 @@ builder_c_t *l_types_builder; /* same as the one on the ffi object */ PyObject *l_dict; /* content, built lazily */ PyObject *l_libname; /* some string that gives the name of the lib */ + PyObject *l_includes; /* tuple of LibObjects included here */ }; #define LibObject_Check(ob) ((Py_TYPE(ob) == &Lib_Type)) @@ -64,6 +65,7 @@ { Py_DECREF(lib->l_dict); Py_DECREF(lib->l_libname); + Py_XDECREF(lib->l_includes); PyObject_Del(lib); } @@ -128,9 +130,11 @@ return NULL; } -static PyObject *lib_build_and_cache_attr(LibObject *lib, PyObject *name) +static PyObject *lib_build_and_cache_attr(LibObject *lib, PyObject *name, + int recursion) { /* does not return a new reference! */ + PyObject *x; char *s = PyText_AsUTF8(name); if (s == NULL) @@ -138,15 +142,45 @@ int index = search_in_globals(&lib->l_types_builder->ctx, s, strlen(s)); if (index < 0) { + + if (lib->l_includes != NULL) { + + if (recursion > 100) { + PyErr_SetString(PyExc_RuntimeError, + "recursion overflow in ffi.include() delegations"); + return NULL; + } + + Py_ssize_t i; + for (i = 0; i < PyTuple_GET_SIZE(lib->l_includes); i++) { + LibObject *lib1; + lib1 = (LibObject *)PyTuple_GET_ITEM(lib->l_includes, i); + x = PyDict_GetItem(lib1->l_dict, name); + if (x != NULL) { + Py_INCREF(x); + goto found; + } + x = lib_build_and_cache_attr(lib1, name, recursion + 1); + if (x != NULL) { + Py_INCREF(x); + goto found; + } + if (PyErr_Occurred()) + return NULL; + } + } + + if (recursion > 0) + return NULL; /* no error set, continue looking elsewhere */ + PyErr_Format(PyExc_AttributeError, - "lib '%.200s' has no function," + "cffi lib '%.200s' has no function," " global variable or constant named '%.200s'", PyText_AS_UTF8(lib->l_libname), s); return NULL; } const struct _cffi_global_s *g = &lib->l_types_builder->ctx.globals[index]; - PyObject *x; CTypeDescrObject *ct; switch (_CFFI_GETOP(g->type_op)) { @@ -217,6 +251,7 @@ return NULL; } + found: if (x != NULL) { int err = PyDict_SetItem(lib->l_dict, name, x); Py_DECREF(x); @@ -230,7 +265,7 @@ { PyObject *x = PyDict_GetItem(lib->l_dict, name); if (x == NULL) { - x = lib_build_and_cache_attr(lib, name); + x = lib_build_and_cache_attr(lib, name, 0); if (x == NULL) return NULL; } @@ -246,7 +281,7 @@ { PyObject *x = PyDict_GetItem(lib->l_dict, name); if (x == NULL) { - x = lib_build_and_cache_attr(lib, name); + x = lib_build_and_cache_attr(lib, name, 0); if (x == NULL) return -1; } @@ -351,5 +386,6 @@ lib->l_types_builder = types_builder; lib->l_dict = dict; lib->l_libname = libname; + lib->l_includes = NULL; return lib; } diff --git a/_cffi1/parse_c_type.h b/_cffi1/parse_c_type.h --- a/_cffi1/parse_c_type.h +++ b/_cffi1/parse_c_type.h @@ -113,7 +113,7 @@ int num_struct_unions; int num_enums; int num_typenames; - const char * const *includes; + const char *const *includes; }; struct _cffi_parse_info_s { diff --git a/_cffi1/test_recompiler.py b/_cffi1/test_recompiler.py --- a/_cffi1/test_recompiler.py +++ b/_cffi1/test_recompiler.py @@ -465,6 +465,17 @@ lib = verify(ffi, "test_include_1", "double ff1(double x) { return 42.5; }") assert lib.ff1(0) == 42.5 +def test_include_1b(): + ffi1 = FFI() + ffi1.cdef("int foo1(int);") + verify(ffi1, "test_include_1b_parent", "int foo1(int x) { return x + 10; }") + ffi = FFI() + ffi.include(ffi1) + ffi.cdef("int foo2(int);") + lib = verify(ffi, "test_include_1b", "int foo2(int x) { return x - 5; }") + assert lib.foo2(42) == 37 + assert lib.foo1(42) == 52 + def test_include_2(): ffi1 = FFI() ffi1.cdef("struct foo_s { int x, y; };") From noreply at buildbot.pypy.org Mon Apr 27 23:48:57 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 27 Apr 2015 23:48:57 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Resolve the F_EXTERNAL struct names by following the ffi.include chain Message-ID: <20150427214857.50AF81C0EC0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1866:8218d7a07822 Date: 2015-04-27 23:48 +0200 http://bitbucket.org/cffi/cffi/changeset/8218d7a07822/ Log: Resolve the F_EXTERNAL struct names by following the ffi.include chain diff --git a/_cffi1/cffi1_module.c b/_cffi1/cffi1_module.c --- a/_cffi1/cffi1_module.c +++ b/_cffi1/cffi1_module.c @@ -114,7 +114,7 @@ if (lib == NULL || PyModule_AddObject(m, "lib", (PyObject *)lib) < 0) return -1; - if (make_included_tuples(ctx->includes, &ffi->included_ffis, + if (make_included_tuples(ctx->includes, &ffi->types_builder->included_ffis, &lib->l_includes) < 0) return -1; diff --git a/_cffi1/cffi_opcode.py b/_cffi1/cffi_opcode.py --- a/_cffi1/cffi_opcode.py +++ b/_cffi1/cffi_opcode.py @@ -100,6 +100,7 @@ F_UNION = 0x01 F_CHECK_FIELDS = 0x02 F_PACKED = 0x04 +F_EXTERNAL = 0x08 CLASS_NAME = {} for _name, _value in globals().items(): diff --git a/_cffi1/ffi_obj.c b/_cffi1/ffi_obj.c --- a/_cffi1/ffi_obj.c +++ b/_cffi1/ffi_obj.c @@ -26,7 +26,6 @@ struct _cffi_parse_info_s info; int ctx_is_static; builder_c_t *types_builder; - PyObject *included_ffis; }; static FFIObject *ffi_internal_new(PyTypeObject *ffitype, @@ -56,7 +55,6 @@ ffi->info.output = internal_output; ffi->info.output_size = FFI_COMPLEXITY_OUTPUT; ffi->ctx_is_static = (static_ctx != NULL); - ffi->included_ffis = NULL; #if 0 ffi->dynamic_types = NULL; #endif @@ -74,15 +72,14 @@ if (!ffi->ctx_is_static) free_builder_c(ffi->types_builder); - Py_XDECREF(ffi->included_ffis); Py_TYPE(ffi)->tp_free((PyObject *)ffi); } static int ffi_traverse(FFIObject *ffi, visitproc visit, void *arg) { Py_VISIT(ffi->types_builder->types_dict); + Py_VISIT(ffi->types_builder->included_ffis); Py_VISIT(ffi->gc_wrefs); - Py_VISIT(ffi->included_ffis); return 0; } @@ -809,3 +806,44 @@ ffiobj_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ }; + + +static PyObject * +_fetch_external_struct_or_union(const struct _cffi_struct_union_s *s, + PyObject *included_ffis, int recursion) +{ + if (included_ffis == NULL) + return NULL; + + if (recursion > 100) { + PyErr_SetString(PyExc_RuntimeError, + "recursion overflow in ffi.include() delegations"); + return NULL; + } + + Py_ssize_t i; + for (i = 0; i < PyTuple_GET_SIZE(included_ffis); i++) { + FFIObject *ffi1; + const struct _cffi_struct_union_s *s1; + int sindex; + PyObject *x; + + ffi1 = (FFIObject *)PyTuple_GET_ITEM(included_ffis, i); + sindex = search_in_struct_unions(&ffi1->types_builder->ctx, s->name, + strlen(s->name)); + if (sindex < 0) /* not found at all */ + continue; + s1 = &ffi1->types_builder->ctx.struct_unions[sindex]; + if ((s1->flags & (_CFFI_F_EXTERNAL | _CFFI_F_UNION)) + == (s->flags & _CFFI_F_UNION)) { + /* s1 is not external, and the same kind (struct or union) as s */ + return _realize_c_struct_or_union(ffi1->types_builder, sindex); + } + /* not found, look more recursively */ + x = _fetch_external_struct_or_union( + s, ffi1->types_builder->included_ffis, recursion + 1); + if (x != NULL || PyErr_Occurred()) + return x; /* either found, or got an error */ + } + return NULL; /* not found at all, leave without an error */ +} diff --git a/_cffi1/realize_c_type.c b/_cffi1/realize_c_type.c --- a/_cffi1/realize_c_type.c +++ b/_cffi1/realize_c_type.c @@ -2,6 +2,7 @@ typedef struct { struct _cffi_type_context_s ctx; /* inlined substructure */ PyObject *types_dict; + PyObject *included_ffis; } builder_c_t; @@ -74,6 +75,9 @@ if (mem[i] != NULL) PyMem_Free((void *)mem[i]); } + + Py_XDECREF(builder->included_ffis); + builder->included_ffis = NULL; } static void free_builder_c(builder_c_t *builder) @@ -101,6 +105,7 @@ memset(&builder->ctx, 0, sizeof(builder->ctx)); builder->types_dict = ldict; + builder->included_ffis = NULL; #if 0 builder->num_types_imported = 0; #endif @@ -268,6 +273,79 @@ } } +static PyObject * /* forward */ +_fetch_external_struct_or_union(const struct _cffi_struct_union_s *s, + PyObject *included_ffis, int recursion); + +static PyObject * +_realize_c_struct_or_union(builder_c_t *builder, int sindex) +{ + PyObject *x; + _cffi_opcode_t op2; + const struct _cffi_struct_union_s *s; + + s = &builder->ctx.struct_unions[sindex]; + op2 = builder->ctx.types[s->type_index]; + if ((((uintptr_t)op2) & 1) == 0) { + x = (PyObject *)op2; /* found already in the "primary" slot */ + Py_INCREF(x); + } + else { + CTypeDescrObject *ct = NULL; + + if (!(s->flags & _CFFI_F_EXTERNAL)) { + int flags = (s->flags & _CFFI_F_UNION) ? CT_UNION : CT_STRUCT; + char *name = alloca(8 + strlen(s->name)); + _realize_name(name, + (s->flags & _CFFI_F_UNION) ? "union " : "struct ", + s->name); + if (strcmp(name, "struct _IO_FILE") == 0) + flags |= CT_IS_FILE; + + x = new_struct_or_union_type(name, flags); + if (x == NULL) + return NULL; + + if (s->first_field_index >= 0) { + ct = (CTypeDescrObject *)x; + ct->ct_size = (Py_ssize_t)s->size; + ct->ct_length = s->alignment; + ct->ct_flags &= ~CT_IS_OPAQUE; + ct->ct_flags |= CT_LAZY_FIELD_LIST; + ct->ct_extra = builder; + } + } + else { + x = _fetch_external_struct_or_union(s, builder->included_ffis, 0); + if (x == NULL) { + if (!PyErr_Occurred()) + PyErr_Format(FFIError, "'%s %.200s' should come from " + "ffi.include() but was not found", + (s->flags & _CFFI_F_UNION) ? "union" + : "struct", s->name); + return NULL; + } + } + + /* Update the "primary" OP_STRUCT_UNION slot */ + assert((((uintptr_t)x) & 1) == 0); + assert(builder->ctx.types[s->type_index] == op2); + Py_INCREF(x); + builder->ctx.types[s->type_index] = x; + + if (ct != NULL && s->size == (size_t)-2) { + /* oops, this struct is unnamed and we couldn't generate + a C expression to get its size. We have to rely on + complete_struct_or_union() to compute it now. */ + if (do_realize_lazy_struct(ct) < 0) { + builder->ctx.types[s->type_index] = op2; + return NULL; + } + } + } + return x; +} + static PyObject * _realize_c_type_or_func(builder_c_t *builder, _cffi_opcode_t opcodes[], int index) @@ -320,62 +398,8 @@ break; case _CFFI_OP_STRUCT_UNION: - { - const struct _cffi_struct_union_s *s; - _cffi_opcode_t op2; - - s = &builder->ctx.struct_unions[_CFFI_GETARG(op)]; - op2 = builder->ctx.types[s->type_index]; - if ((((uintptr_t)op2) & 1) == 0) { - x = (PyObject *)op2; - Py_INCREF(x); - } - else { - int flags = (s->flags & _CFFI_F_UNION) ? CT_UNION : CT_STRUCT; - char *name = alloca(8 + strlen(s->name)); - _realize_name(name, - (s->flags & _CFFI_F_UNION) ? "union " : "struct ", - s->name); - if (strcmp(name, "struct _IO_FILE") == 0) - flags |= CT_IS_FILE; - - x = new_struct_or_union_type(name, flags); - - CTypeDescrObject *ct = NULL; - if (s->first_field_index >= 0) { - ct = (CTypeDescrObject *)x; - ct->ct_size = (Py_ssize_t)s->size; - ct->ct_length = s->alignment; - ct->ct_flags &= ~CT_IS_OPAQUE; - ct->ct_flags |= CT_LAZY_FIELD_LIST; - ct->ct_extra = builder; - } - - /* Update the "primary" OP_STRUCT_UNION slot, which - may be the same or a different slot than the "current" one */ - assert((((uintptr_t)x) & 1) == 0); - assert(builder->ctx.types[s->type_index] == op2); - Py_INCREF(x); - builder->ctx.types[s->type_index] = x; - - if (s->size == (size_t)-2) { - /* oops, this struct is unnamed and we couldn't generate - a C expression to get its size. We have to rely on - complete_struct_or_union() to compute it now. */ - assert(ct != NULL); - if (do_realize_lazy_struct(ct) < 0) { - builder->ctx.types[s->type_index] = op2; - return NULL; - } - } - - /* Done, leave without updating the "current" slot because - it may be done already above. If not, never mind, the - next call to realize_c_type() will do it. */ - return x; - } + x = _realize_c_struct_or_union(builder, _CFFI_GETARG(op)); break; - } case _CFFI_OP_ENUM: { @@ -530,7 +554,7 @@ return NULL; } - if (x != NULL && opcodes == builder->ctx.types) { + if (x != NULL && opcodes == builder->ctx.types && opcodes[index] != x) { assert((((uintptr_t)x) & 1) == 0); assert((((uintptr_t)opcodes[index]) & 1) == 1); Py_INCREF(x); diff --git a/_cffi1/test_recompiler.py b/_cffi1/test_recompiler.py --- a/_cffi1/test_recompiler.py +++ b/_cffi1/test_recompiler.py @@ -484,7 +484,7 @@ ffi.include(ffi1) ffi.cdef("struct foo_s *ff2(struct foo_s *);") lib = verify(ffi, "test_include_2", - "struct foo_s { int x, y; };\n" + "struct foo_s { int x, y; }; //usually from a #include\n" "struct foo_s *ff2(struct foo_s *p) { p->y++; return p; }") p = ffi.new("struct foo_s *") p.y = 41 From noreply at buildbot.pypy.org Mon Apr 27 23:48:58 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 27 Apr 2015 23:48:58 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: merge heads Message-ID: <20150427214858.4FF6E1C0EC0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1867:292cb69ac76c Date: 2015-04-27 23:49 +0200 http://bitbucket.org/cffi/cffi/changeset/292cb69ac76c/ Log: merge heads diff --git a/_cffi1/lib_obj.c b/_cffi1/lib_obj.c --- a/_cffi1/lib_obj.c +++ b/_cffi1/lib_obj.c @@ -231,8 +231,7 @@ _CFFI_GETARG(g->type_op)); if (ct == NULL) return NULL; - if (g->size != ct->ct_size && - g->size != (size_t)-1 && ct->ct_size != -1) { + if (g->size != ct->ct_size && g->size != 0 && ct->ct_size > 0) { PyErr_Format(FFIError, "global variable '%.200s' should be %zd bytes " "according to the cdef, but is actually %zd", diff --git a/_cffi1/parse_c_type.h b/_cffi1/parse_c_type.h --- a/_cffi1/parse_c_type.h +++ b/_cffi1/parse_c_type.h @@ -63,8 +63,8 @@ struct _cffi_global_s { const char *name; void *address; - size_t size; // -1 if unknown _cffi_opcode_t type_op; + size_t size; // 0 if unknown }; struct _cffi_struct_union_s { diff --git a/_cffi1/recompiler.py b/_cffi1/recompiler.py --- a/_cffi1/recompiler.py +++ b/_cffi1/recompiler.py @@ -434,8 +434,7 @@ else: meth_kind = 'V' # 'METH_VARARGS' self._lsts["global"].append( - ' { "%s", _cffi_f_%s, (size_t)-1, ' - '_CFFI_OP(_CFFI_OP_CPYTHON_BLTN_%s, %d) },' + ' { "%s", _cffi_f_%s, _CFFI_OP(_CFFI_OP_CPYTHON_BLTN_%s, %d), 0 },' % (name, name, meth_kind, type_index)) # ---------- @@ -654,8 +653,7 @@ type_index = self._typesdict[tp] type_op = '_CFFI_OP(_CFFI_OP_CONSTANT, %d)' % type_index self._lsts["global"].append( - ' { "%s", _cffi_const_%s, (size_t)-1, %s },' % - (name, name, type_op)) + ' { "%s", _cffi_const_%s, %s, 0 },' % (name, name, type_op)) # ---------- # enums @@ -672,7 +670,7 @@ type_op = '_CFFI_OP(_CFFI_OP_ENUM, -1)' for enumerator in tp.enumerators: self._lsts["global"].append( - ' { "%s", _cffi_const_%s, (size_t)-1, %s },' % + ' { "%s", _cffi_const_%s, %s, 0 },' % (enumerator, enumerator, type_op)) # if cname is not None and '$' not in cname: @@ -703,9 +701,8 @@ def _generate_cpy_macro_ctx(self, tp, name): self._lsts["global"].append( - ' { "%s", _cffi_const_%s, (size_t)-1,' - ' _CFFI_OP(_CFFI_OP_CONSTANT_INT, 0) },' % - (name, name)) + ' { "%s", _cffi_const_%s,' + ' _CFFI_OP(_CFFI_OP_CONSTANT_INT, 0), 0 },' % (name, name)) # ---------- # global variables @@ -728,10 +725,10 @@ if tp.sizeof_enabled(): size = "sizeof(%s)" % (name,) else: - size = "(size_t)-1" + size = "0" self._lsts["global"].append( - ' { "%s", &%s, %s, _CFFI_OP(_CFFI_OP_GLOBAL_VAR, %d)},' - % (name, name, size, type_index)) + ' { "%s", &%s, _CFFI_OP(_CFFI_OP_GLOBAL_VAR, %d), %s },' + % (name, name, type_index, size)) # ---------- # emitting the opcodes for individual types diff --git a/cffi/model.py b/cffi/model.py --- a/cffi/model.py +++ b/cffi/model.py @@ -105,8 +105,26 @@ 'uint32_t': 'i', 'int64_t': 'i', 'uint64_t': 'i', + 'int_least8_t': 'i', + 'uint_least8_t': 'i', + 'int_least16_t': 'i', + 'uint_least16_t': 'i', + 'int_least32_t': 'i', + 'uint_least32_t': 'i', + 'int_least64_t': 'i', + 'uint_least64_t': 'i', + 'int_fast8_t': 'i', + 'uint_fast8_t': 'i', + 'int_fast16_t': 'i', + 'uint_fast16_t': 'i', + 'int_fast32_t': 'i', + 'uint_fast32_t': 'i', + 'int_fast64_t': 'i', + 'uint_fast64_t': 'i', 'intptr_t': 'i', 'uintptr_t': 'i', + 'intmax_t': 'i', + 'uintmax_t': 'i', 'ptrdiff_t': 'i', 'size_t': 'i', 'ssize_t': 'i', diff --git a/testing/backend_tests.py b/testing/backend_tests.py --- a/testing/backend_tests.py +++ b/testing/backend_tests.py @@ -1703,5 +1703,3 @@ assert lib.DOT_HEX == 0x100 assert lib.DOT_HEX2 == 0x10 assert lib.DOT_UL == 1000 - - diff --git a/testing/test_ffi_backend.py b/testing/test_ffi_backend.py --- a/testing/test_ffi_backend.py +++ b/testing/test_ffi_backend.py @@ -222,3 +222,57 @@ assert ffi.typeof(c) is ffi.typeof("char[]") ffi.cast("unsigned short *", c)[1] += 500 assert list(a) == [10000, 20500, 30000] + + def test_all_primitives(self): + ffi = FFI() + for name in [ + "char", + "short", + "int", + "long", + "long long", + "signed char", + "unsigned char", + "unsigned short", + "unsigned int", + "unsigned long", + "unsigned long long", + "float", + "double", + "long double", + "wchar_t", + "_Bool", + "int8_t", + "uint8_t", + "int16_t", + "uint16_t", + "int32_t", + "uint32_t", + "int64_t", + "uint64_t", + "int_least8_t", + "uint_least8_t", + "int_least16_t", + "uint_least16_t", + "int_least32_t", + "uint_least32_t", + "int_least64_t", + "uint_least64_t", + "int_fast8_t", + "uint_fast8_t", + "int_fast16_t", + "uint_fast16_t", + "int_fast32_t", + "uint_fast32_t", + "int_fast64_t", + "uint_fast64_t", + "intptr_t", + "uintptr_t", + "intmax_t", + "uintmax_t", + "ptrdiff_t", + "size_t", + "ssize_t", + ]: + x = ffi.sizeof(name) + assert 1 <= x <= 16 From noreply at buildbot.pypy.org Tue Apr 28 02:15:05 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 28 Apr 2015 02:15:05 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Add the 0.9.3 integer types Message-ID: <20150428001505.3FCBB1C114E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1868:2fa0c70b8db6 Date: 2015-04-28 02:15 +0200 http://bitbucket.org/cffi/cffi/changeset/2fa0c70b8db6/ Log: Add the 0.9.3 integer types diff --git a/_cffi1/cffi_opcode.py b/_cffi1/cffi_opcode.py --- a/_cffi1/cffi_opcode.py +++ b/_cffi1/cffi_opcode.py @@ -62,8 +62,26 @@ PRIM_PTRDIFF = 27 PRIM_SIZE = 28 PRIM_SSIZE = 29 +PRIM_INT_LEAST8 = 30 +PRIM_UINT_LEAST8 = 31 +PRIM_INT_LEAST16 = 32 +PRIM_UINT_LEAST16 = 33 +PRIM_INT_LEAST32 = 34 +PRIM_UINT_LEAST32 = 35 +PRIM_INT_LEAST64 = 36 +PRIM_UINT_LEAST64 = 37 +PRIM_INT_FAST8 = 38 +PRIM_UINT_FAST8 = 39 +PRIM_INT_FAST16 = 40 +PRIM_UINT_FAST16 = 41 +PRIM_INT_FAST32 = 42 +PRIM_UINT_FAST32 = 43 +PRIM_INT_FAST64 = 44 +PRIM_UINT_FAST64 = 45 +PRIM_INTMAX = 46 +PRIM_UINTMAX = 47 -_NUM_PRIM = 30 +_NUM_PRIM = 48 PRIMITIVE_TO_INDEX = { 'char': PRIM_CHAR, @@ -95,6 +113,24 @@ 'ptrdiff_t': PRIM_PTRDIFF, 'size_t': PRIM_SIZE, 'ssize_t': PRIM_SSIZE, + 'int_least8_t': PRIM_INT_LEAST8, + 'uint_least8_t': PRIM_UINT_LEAST8, + 'int_least16_t': PRIM_INT_LEAST16, + 'uint_least16_t': PRIM_UINT_LEAST16, + 'int_least32_t': PRIM_INT_LEAST32, + 'uint_least32_t': PRIM_UINT_LEAST32, + 'int_least64_t': PRIM_INT_LEAST64, + 'uint_least64_t': PRIM_UINT_LEAST64, + 'int_fast8_t': PRIM_INT_FAST8, + 'uint_fast8_t': PRIM_UINT_FAST8, + 'int_fast16_t': PRIM_INT_FAST16, + 'uint_fast16_t': PRIM_UINT_FAST16, + 'int_fast32_t': PRIM_INT_FAST32, + 'uint_fast32_t': PRIM_UINT_FAST32, + 'int_fast64_t': PRIM_INT_FAST64, + 'uint_fast64_t': PRIM_UINT_FAST64, + 'intmax_t': PRIM_INTMAX, + 'uintmax_t': PRIM_UINTMAX, } F_UNION = 0x01 diff --git a/_cffi1/parse_c_type.c b/_cffi1/parse_c_type.c --- a/_cffi1/parse_c_type.c +++ b/_cffi1/parse_c_type.c @@ -459,14 +459,36 @@ if (size == 7 && !memcmp(p, "uint8", 5)) return _CFFI_PRIM_UINT8; break; + case 'a': + if (size == 8 && !memcmp(p, "intmax", 6)) return _CFFI_PRIM_INTMAX; + break; + case 'e': if (size == 7 && !memcmp(p, "ssize", 5)) return _CFFI_PRIM_SSIZE; break; + case 'f': + if (size == 11 && !memcmp(p, "int_fast8", 9)) return _CFFI_PRIM_INT_FAST8; + if (size == 12 && !memcmp(p, "int_fast16", 10)) return _CFFI_PRIM_INT_FAST16; + if (size == 12 && !memcmp(p, "int_fast32", 10)) return _CFFI_PRIM_INT_FAST32; + if (size == 12 && !memcmp(p, "int_fast64", 10)) return _CFFI_PRIM_INT_FAST64; + break; + case 'i': if (size == 9 && !memcmp(p, "ptrdiff", 7)) return _CFFI_PRIM_PTRDIFF; break; + case 'l': + if (size == 12 && !memcmp(p, "int_least8", 10)) return _CFFI_PRIM_INT_LEAST8; + if (size == 13 && !memcmp(p, "int_least16", 11)) return _CFFI_PRIM_INT_LEAST16; + if (size == 13 && !memcmp(p, "int_least32", 11)) return _CFFI_PRIM_INT_LEAST32; + if (size == 13 && !memcmp(p, "int_least64", 11)) return _CFFI_PRIM_INT_LEAST64; + break; + + case 'm': + if (size == 9 && !memcmp(p, "uintmax", 7)) return _CFFI_PRIM_UINTMAX; + break; + case 'p': if (size == 9 && !memcmp(p, "uintptr", 7)) return _CFFI_PRIM_UINTPTR; break; @@ -482,6 +504,34 @@ case '_': if (size == 6 && !memcmp(p, "size", 4)) return _CFFI_PRIM_SIZE; if (size == 6 && !memcmp(p, "int8", 4)) return _CFFI_PRIM_INT8; + if (size >= 12) { + switch (p[10]) { + case '1': + if (size == 14 && !memcmp(p, "uint_least16", 12)) return _CFFI_PRIM_UINT_LEAST16; + break; + case '2': + if (size == 13 && !memcmp(p, "uint_fast32", 11)) return _CFFI_PRIM_UINT_FAST32; + break; + case '3': + if (size == 14 && !memcmp(p, "uint_least32", 12)) return _CFFI_PRIM_UINT_LEAST32; + break; + case '4': + if (size == 13 && !memcmp(p, "uint_fast64", 11)) return _CFFI_PRIM_UINT_FAST64; + break; + case '6': + if (size == 14 && !memcmp(p, "uint_least64", 12)) return _CFFI_PRIM_UINT_LEAST64; + if (size == 13 && !memcmp(p, "uint_fast16", 11)) return _CFFI_PRIM_UINT_FAST16; + break; + case '8': + if (size == 13 && !memcmp(p, "uint_least8", 11)) return _CFFI_PRIM_UINT_LEAST8; + break; + case '_': + if (size == 12 && !memcmp(p, "uint_fast8", 10)) return _CFFI_PRIM_UINT_FAST8; + break; + default: + break; + } + } break; default: diff --git a/_cffi1/parse_c_type.h b/_cffi1/parse_c_type.h --- a/_cffi1/parse_c_type.h +++ b/_cffi1/parse_c_type.h @@ -56,8 +56,26 @@ #define _CFFI_PRIM_PTRDIFF 27 #define _CFFI_PRIM_SIZE 28 #define _CFFI_PRIM_SSIZE 29 +#define _CFFI_PRIM_INT_LEAST8 30 +#define _CFFI_PRIM_UINT_LEAST8 31 +#define _CFFI_PRIM_INT_LEAST16 32 +#define _CFFI_PRIM_UINT_LEAST16 33 +#define _CFFI_PRIM_INT_LEAST32 34 +#define _CFFI_PRIM_UINT_LEAST32 35 +#define _CFFI_PRIM_INT_LEAST64 36 +#define _CFFI_PRIM_UINT_LEAST64 37 +#define _CFFI_PRIM_INT_FAST8 38 +#define _CFFI_PRIM_UINT_FAST8 39 +#define _CFFI_PRIM_INT_FAST16 40 +#define _CFFI_PRIM_UINT_FAST16 41 +#define _CFFI_PRIM_INT_FAST32 42 +#define _CFFI_PRIM_UINT_FAST32 43 +#define _CFFI_PRIM_INT_FAST64 44 +#define _CFFI_PRIM_UINT_FAST64 45 +#define _CFFI_PRIM_INTMAX 46 +#define _CFFI_PRIM_UINTMAX 47 -#define _CFFI__NUM_PRIM 30 +#define _CFFI__NUM_PRIM 48 struct _cffi_global_s { diff --git a/_cffi1/realize_c_type.c b/_cffi1/realize_c_type.c --- a/_cffi1/realize_c_type.c +++ b/_cffi1/realize_c_type.c @@ -146,6 +146,24 @@ "ptrdiff_t", "size_t", "ssize_t", + "int_least8_t", + "uint_least8_t", + "int_least16_t", + "uint_least16_t", + "int_least32_t", + "uint_least32_t", + "int_least64_t", + "uint_least64_t", + "int_fast8_t", + "uint_fast8_t", + "int_fast16_t", + "uint_fast16_t", + "int_fast32_t", + "uint_fast32_t", + "int_fast64_t", + "uint_fast64_t", + "intmax_t", + "uintmax_t", }; PyObject *x; diff --git a/_cffi1/test_new_ffi_1.py b/_cffi1/test_new_ffi_1.py --- a/_cffi1/test_new_ffi_1.py +++ b/_cffi1/test_new_ffi_1.py @@ -1613,3 +1613,58 @@ assert ffi.typeof(c) is ffi.typeof("char[]") ffi.cast("unsigned short *", c)[1] += 500 assert list(a) == [10000, 20500, 30000] + + def test_all_primitives(self): + from .cffi_opcode import PRIMITIVE_TO_INDEX + assert set(PRIMITIVE_TO_INDEX) == set([ + "char", + "short", + "int", + "long", + "long long", + "signed char", + "unsigned char", + "unsigned short", + "unsigned int", + "unsigned long", + "unsigned long long", + "float", + "double", + "long double", + "wchar_t", + "_Bool", + "int8_t", + "uint8_t", + "int16_t", + "uint16_t", + "int32_t", + "uint32_t", + "int64_t", + "uint64_t", + "int_least8_t", + "uint_least8_t", + "int_least16_t", + "uint_least16_t", + "int_least32_t", + "uint_least32_t", + "int_least64_t", + "uint_least64_t", + "int_fast8_t", + "uint_fast8_t", + "int_fast16_t", + "uint_fast16_t", + "int_fast32_t", + "uint_fast32_t", + "int_fast64_t", + "uint_fast64_t", + "intptr_t", + "uintptr_t", + "intmax_t", + "uintmax_t", + "ptrdiff_t", + "size_t", + "ssize_t", + ]) + for name in PRIMITIVE_TO_INDEX: + x = ffi.sizeof(name) + assert 1 <= x <= 16 From noreply at buildbot.pypy.org Tue Apr 28 11:30:47 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 28 Apr 2015 11:30:47 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Check again the value of #define constants if they are not defined with Message-ID: <20150428093047.498581C1439@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1869:1b74d35a49ed Date: 2015-04-28 11:31 +0200 http://bitbucket.org/cffi/cffi/changeset/1b74d35a49ed/ Log: Check again the value of #define constants if they are not defined with '...'. diff --git a/_cffi1/_cffi_include.h b/_cffi1/_cffi_include.h --- a/_cffi1/_cffi_include.h +++ b/_cffi1/_cffi_include.h @@ -170,6 +170,10 @@ (size) == 8 ? ((sign) ? _CFFI_PRIM_INT64 : _CFFI_PRIM_UINT64) : \ 0) +#define _cffi_check_int(got, got_nonpos, expected) \ + ((got_nonpos) == (expected <= 0) && \ + (got) == (unsigned long long)expected) + static int _cffi_init(void) { diff --git a/_cffi1/realize_c_type.c b/_cffi1/realize_c_type.c --- a/_cffi1/realize_c_type.c +++ b/_cffi1/realize_c_type.c @@ -186,24 +186,37 @@ static PyObject *realize_global_int(const struct _cffi_global_s *g) { - PyObject *x; unsigned long long value; /* note: we cast g->address to this function type; we do the same in parse_c_type:parse_sequel() too */ int neg = ((int(*)(unsigned long long*))g->address)(&value); - if (!neg) { + + switch (neg) { + + case 0: if (value <= (unsigned long long)LONG_MAX) - x = PyInt_FromLong((long)value); + return PyInt_FromLong((long)value); else - x = PyLong_FromUnsignedLongLong(value); + return PyLong_FromUnsignedLongLong(value); + + case 1: + if ((long long)value >= (long long)LONG_MIN) + return PyInt_FromLong((long)value); + else + return PyLong_FromLongLong((long long)value); + + default: + break; } - else { - if ((long long)value >= (long long)LONG_MIN) - x = PyInt_FromLong((long)value); - else - x = PyLong_FromLongLong((long long)value); - } - return x; + + char got[64]; + if (neg == 2) + sprintf(got, "%llu (0x%llx)", value, value); + else + sprintf(got, "%lld", (long long)value); + PyErr_Format(FFIError, "the C compiler says '%.200s' is equal to %s, " + "but the cdef disagrees", g->name, got); + return NULL; } static PyObject * @@ -462,6 +475,8 @@ while (p[j] != ',' && p[j] != '\0') j++; tmp = PyString_FromStringAndSize(p, j); + if (tmp == NULL) + break; PyTuple_SET_ITEM(enumerators, i, tmp); gindex = search_in_globals(&builder->ctx, p, j); @@ -470,6 +485,8 @@ assert(g->type_op == _CFFI_OP(_CFFI_OP_ENUM, -1)); tmp = realize_global_int(g); + if (tmp == NULL) + break; PyTuple_SET_ITEM(enumvalues, i, tmp); p += j + 1; diff --git a/_cffi1/recompiler.py b/_cffi1/recompiler.py --- a/_cffi1/recompiler.py +++ b/_cffi1/recompiler.py @@ -615,7 +615,8 @@ # ---------- # constants, declared with "static const ..." - def _generate_cpy_const(self, is_int, name, tp=None, category='const'): + def _generate_cpy_const(self, is_int, name, tp=None, category='const', + check_value=None): if (category, name) in self._seen_constants: raise ffiplatform.VerificationError( "duplicate declaration of %s '%s'" % (category, name)) @@ -626,11 +627,16 @@ if is_int: prnt('static int %s(unsigned long long *o)' % funcname) prnt('{') + prnt(' int n = (%s) <= 0;' % (name,)) prnt(' *o = (unsigned long long)((%s) << 0);' ' /* check that we get an integer */' % (name,)) - prnt(' return (%s) <= 0;' % (name,)) + if check_value is not None: + prnt(' if (!_cffi_check_int(*o, n, %s))' % (check_value,)) + prnt(' n |= 2;') + prnt(' return n;') prnt('}') else: + assert check_value is None prnt('static void %s(char *o)' % funcname) prnt('{') prnt(' *(%s)o = %s;' % (tp.get_c_name('*'), name)) @@ -695,9 +701,11 @@ pass def _generate_cpy_macro_decl(self, tp, name): - # for now, we ignore the value (if != ',,,') given in the cdef - # and always trust the value coming from the C compiler - self._generate_cpy_const(True, name) + if tp == '...': + check_value = None + else: + check_value = tp # an integer + self._generate_cpy_const(True, name, check_value=check_value) def _generate_cpy_macro_ctx(self, tp, name): self._lsts["global"].append( diff --git a/_cffi1/test_recompiler.py b/_cffi1/test_recompiler.py --- a/_cffi1/test_recompiler.py +++ b/_cffi1/test_recompiler.py @@ -175,17 +175,40 @@ assert lib.FOOBAR == -6912 py.test.raises(AttributeError, "lib.FOOBAR = 2") -def test_macro_check_value_ok(): +def test_macro_check_value(): + # the value '-0x80000000' in C sources does not have a clear meaning + # to me; it appears to have a different effect than '-2147483648'... + vals = ['42', '-42', '0x80000000', '-2147483648', + '0', '9223372036854775809ULL', + '-9223372036854775807LL'] ffi = FFI() - ffi.cdef("#define FOOBAR 42") - lib = verify(ffi, 'test_macro_check_value_ok', "#define FOOBAR 42") - assert lib.FOOBAR == 42 + cdef_lines = ['#define FOO_%d_%d %s' % (i, j, vals[i]) + for i in range(len(vals)) + for j in range(len(vals))] + ffi.cdef('\n'.join(cdef_lines)) -def test_macro_check_value_fail(): - ffi = FFI() - ffi.cdef("#define FOOBAR 42") - lib = verify(ffi, 'test_macro_check_value_fail', "#define FOOBAR 43") - assert lib.FOOBAR == 43 # for now, we don't check the cdef value + verify_lines = ['#define FOO_%d_%d %s' % (i, j, vals[j]) # [j], not [i] + for i in range(len(vals)) + for j in range(len(vals))] + lib = verify(ffi, 'test_macro_check_value_ok', + '\n'.join(verify_lines)) + # + for j in range(len(vals)): + c_got = int(vals[j].replace('U', '').replace('L', ''), 0) + c_compiler_msg = str(c_got) + if c_got > 0: + c_compiler_msg += ' (0x%x)' % (c_got,) + # + for i in range(len(vals)): + attrname = 'FOO_%d_%d' % (i, j) + if i == j: + x = getattr(lib, attrname) + assert x == c_got + else: + e = py.test.raises(ffi.error, getattr, lib, attrname) + assert str(e.value) == ( + "the C compiler says '%s' is equal to " + "%s, but the cdef disagrees" % (attrname, c_compiler_msg)) def test_constant(): ffi = FFI() diff --git a/_cffi1/test_verify1.py b/_cffi1/test_verify1.py --- a/_cffi1/test_verify1.py +++ b/_cffi1/test_verify1.py @@ -2171,4 +2171,6 @@ ffi = FFI() ffi.cdef("#define FOO 123") lib = ffi.verify("#define FOO 124") # used to complain - assert lib.FOO == 124 + e = py.test.raises(ffi.error, "lib.FOO") + assert str(e.value) == ("the C compiler says 'FOO' is equal to 124 (0x7c)," + " but the cdef disagrees") diff --git a/cffi/cparser.py b/cffi/cparser.py --- a/cffi/cparser.py +++ b/cffi/cparser.py @@ -23,7 +23,7 @@ _r_partial_array = re.compile(r"\[\s*\.\.\.\s*\]") _r_words = re.compile(r"\w+|\S") _parser_cache = None -_r_int_literal = re.compile(r"^0?x?[0-9a-f]+u?l?$", re.IGNORECASE) +_r_int_literal = re.compile(r"^0?x?[0-9a-f]+[lu]*$", re.IGNORECASE) def _get_parser(): global _parser_cache @@ -218,6 +218,9 @@ def _process_macros(self, macros): for key, value in macros.items(): value = value.strip() + neg = value.startswith('-') + if neg: + value = value[1:].strip() match = _r_int_literal.search(value) if match is not None: int_str = match.group(0).lower().rstrip("ul") @@ -229,6 +232,7 @@ int_str = "0o" + int_str[1:] pyvalue = int(int_str, 0) + if neg: pyvalue = -pyvalue self._add_constants(key, pyvalue) self._declare('macro ' + key, pyvalue) elif value == '...': From noreply at buildbot.pypy.org Tue Apr 28 11:44:41 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 28 Apr 2015 11:44:41 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Fix the test, still xfail Message-ID: <20150428094441.8A8571C083B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1870:b8665b33150c Date: 2015-04-28 11:45 +0200 http://bitbucket.org/cffi/cffi/changeset/b8665b33150c/ Log: Fix the test, still xfail diff --git a/_cffi1/ffi_obj.c b/_cffi1/ffi_obj.c --- a/_cffi1/ffi_obj.c +++ b/_cffi1/ffi_obj.c @@ -766,7 +766,7 @@ static PyTypeObject FFI_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "FFI", + "CompiledFFI", sizeof(FFIObject), 0, (destructor)ffi_dealloc, /* tp_dealloc */ diff --git a/_cffi1/lib_obj.c b/_cffi1/lib_obj.c --- a/_cffi1/lib_obj.c +++ b/_cffi1/lib_obj.c @@ -328,7 +328,7 @@ static PyTypeObject Lib_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "Lib", + "CompiledLib", sizeof(LibObject), 0, (destructor)lib_dealloc, /* tp_dealloc */ diff --git a/_cffi1/test_new_ffi_1.py b/_cffi1/test_new_ffi_1.py --- a/_cffi1/test_new_ffi_1.py +++ b/_cffi1/test_new_ffi_1.py @@ -14,7 +14,7 @@ def setup_module(): - global ffi + global ffi, ffi1 ffi1 = cffi.FFI() DEFS = r""" struct repr { short a, b, c; }; @@ -1500,64 +1500,32 @@ assert foo2.a == 20 assert foo2.b == 30 - def test_missing_include(self): + def test_include_struct_union_enum_typedef(self): py.test.xfail("ffi.include") - backend = self.Backend() - ffi1 = FFI(backend=backend) - ffi2 = FFI(backend=backend) - ffi1.cdef("typedef signed char schar_t;") - py.test.raises(CDefError, ffi2.cast, "schar_t", 142) - - def test_include_typedef(self): - py.test.xfail("ffi.include") - backend = self.Backend() - ffi1 = FFI(backend=backend) - ffi2 = FFI(backend=backend) - ffi1.cdef("typedef signed char schar_t;") + ffi2 = cffi.FFI() ffi2.include(ffi1) - p = ffi2.cast("schar_t", 142) - assert int(p) == 142 - 256 - - def test_include_struct(self): - py.test.xfail("ffi.include") - backend = self.Backend() - ffi1 = FFI(backend=backend) - ffi2 = FFI(backend=backend) - ffi1.cdef("struct foo { int x; };") - ffi2.include(ffi1) - p = ffi2.new("struct foo *", [142]) - assert p.x == 142 - - def test_include_union(self): - py.test.xfail("ffi.include") - backend = self.Backend() - ffi1 = FFI(backend=backend) - ffi2 = FFI(backend=backend) - ffi1.cdef("union foo { int x; };") - ffi2.include(ffi1) - p = ffi2.new("union foo *", [142]) - assert p.x == 142 - - def test_include_enum(self): - py.test.xfail("ffi.include") - backend = self.Backend() - ffi1 = FFI(backend=backend) - ffi2 = FFI(backend=backend) - ffi1.cdef("enum foo { FA, FB, FC };") - ffi2.include(ffi1) - p = ffi2.cast("enum foo", 1) - assert ffi2.string(p) == "FB" - assert ffi2.sizeof("char[FC]") == 2 - - def test_include_typedef_2(self): - py.test.xfail("ffi.include") - backend = self.Backend() - ffi1 = FFI(backend=backend) - ffi2 = FFI(backend=backend) - ffi1.cdef("typedef struct { int x; } *foo_p;") - ffi2.include(ffi1) - p = ffi2.new("foo_p", [142]) - assert p.x == 142 + outputfilename = recompile(ffi2, + "test_include_struct_union_enum_typedef", + "", tmpdir=str(udir)) + module = imp.load_dynamic("test_include_struct_union_enum_typedef", + outputfilename) + ffi2 = module.ffi + # + p = ffi2.new("struct nonpacked *", ['A', -43141]) + assert p.a == 'A' + assert p.b == -43141 + # + p = ffi.new("union simple_u", [-52525]) + assert p.a == -52525 + # + p = ffi.cast("enum foq", 2) + assert ffi.string(p) == "CC0" + assert ffi2.sizeof("char[CC0]") == 2 + # + p = ffi.new("anon_foo_t *", [-52526]) + assert p.a == -52526 + p = ffi.new("named_foo_p", [-52527]) + assert p.a == -52527 def test_struct_packed(self): # struct nonpacked { char a; int b; }; diff --git a/cffi/api.py b/cffi/api.py --- a/cffi/api.py +++ b/cffi/api.py @@ -419,6 +419,10 @@ variables, which must anyway be accessed directly from the lib object returned by the original FFI instance. """ + if not isinstance(ffi_to_include, FFI): + raise TypeError("ffi.include() expects an argument that is also of" + " type cffi.FFI, not %r" % ( + type(ffi_to_include).__name__,)) with ffi_to_include._lock: with self._lock: self._parser.include(ffi_to_include._parser) From noreply at buildbot.pypy.org Tue Apr 28 12:12:03 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 28 Apr 2015 12:12:03 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Fix the test, now it passes Message-ID: <20150428101203.94E6A1C04A4@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1872:c5891bfec7b0 Date: 2015-04-28 12:12 +0200 http://bitbucket.org/cffi/cffi/changeset/c5891bfec7b0/ Log: Fix the test, now it passes diff --git a/_cffi1/test_new_ffi_1.py b/_cffi1/test_new_ffi_1.py --- a/_cffi1/test_new_ffi_1.py +++ b/_cffi1/test_new_ffi_1.py @@ -14,7 +14,7 @@ def setup_module(): - global ffi, ffi1 + global ffi, construction_params ffi1 = cffi.FFI() DEFS = r""" struct repr { short a, b, c; }; @@ -83,6 +83,7 @@ tmpdir=str(udir)) module = imp.load_dynamic("test_new_ffi_1", outputfilename) ffi = module.ffi + construction_params = (ffi1, CCODE) class TestNewFFI1: @@ -1501,12 +1502,13 @@ assert foo2.b == 30 def test_include_struct_union_enum_typedef(self): - py.test.xfail("ffi.include") + #py.test.xfail("ffi.include") + ffi1, CCODE = construction_params ffi2 = cffi.FFI() ffi2.include(ffi1) outputfilename = recompile(ffi2, "test_include_struct_union_enum_typedef", - "", tmpdir=str(udir)) + CCODE, tmpdir=str(udir)) module = imp.load_dynamic("test_include_struct_union_enum_typedef", outputfilename) ffi2 = module.ffi @@ -1515,7 +1517,7 @@ assert p.a == 'A' assert p.b == -43141 # - p = ffi.new("union simple_u", [-52525]) + p = ffi.new("union simple_u *", [-52525]) assert p.a == -52525 # p = ffi.cast("enum foq", 2) From noreply at buildbot.pypy.org Tue Apr 28 12:12:02 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 28 Apr 2015 12:12:02 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: tests and one fix Message-ID: <20150428101202.7DDBC1C04A4@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1871:f6e9b11edb1f Date: 2015-04-28 12:08 +0200 http://bitbucket.org/cffi/cffi/changeset/f6e9b11edb1f/ Log: tests and one fix diff --git a/_cffi1/test_recompiler.py b/_cffi1/test_recompiler.py --- a/_cffi1/test_recompiler.py +++ b/_cffi1/test_recompiler.py @@ -514,3 +514,65 @@ q = lib.ff2(p) assert q == p assert p.y == 42 + +def test_include_3(): + ffi1 = FFI() + ffi1.cdef("typedef short sshort_t;") + verify(ffi1, "test_include_3_parent", "typedef short sshort_t;") + ffi = FFI() + ffi.include(ffi1) + ffi.cdef("sshort_t ff3(sshort_t);") + lib = verify(ffi, "test_include_3", + "typedef short sshort_t; //usually from a #include\n" + "sshort_t ff3(sshort_t x) { return x + 42; }") + assert lib.ff3(10) == 52 + assert ffi.typeof(ffi.cast("sshort_t", 42)) is ffi.typeof("short") + +def test_include_4(): + ffi1 = FFI() + ffi1.cdef("typedef struct { int x; } mystruct_t;") + verify(ffi1, "test_include_4_parent", + "typedef struct { int x; } mystruct_t;") + ffi = FFI() + ffi.include(ffi1) + ffi.cdef("mystruct_t *ff4(mystruct_t *);") + lib = verify(ffi, "test_include_4", + "typedef struct {int x; } mystruct_t; //usually from a #include\n" + "mystruct_t *ff4(mystruct_t *p) { p->x += 42; return p; }") + p = ffi.new("mystruct_t *", [10]) + q = lib.ff4(p) + assert q == p + assert p.x == 52 + +def test_include_5(): + py.test.xfail("also fails in 0.9.3") + ffi1 = FFI() + ffi1.cdef("typedef struct { int x; } *mystruct_p;") + verify(ffi1, "test_include_5_parent", + "typedef struct { int x; } *mystruct_p;") + ffi = FFI() + ffi.include(ffi1) + ffi.cdef("mystruct_p ff5(mystruct_p);") + lib = verify(ffi, "test_include_5", + "typedef struct {int x; } *mystruct_p; //usually from a #include\n" + "mystruct_p ff5(mystruct_p p) { p->x += 42; return p; }") + p = ffi.new("mystruct_p", [10]) + q = lib.ff5(p) + assert q == p + assert p.x == 52 + +def test_include_6(): + ffi1 = FFI() + ffi1.cdef("typedef ... mystruct_t;") + verify(ffi1, "test_include_6_parent", + "typedef struct _mystruct_s mystruct_t;") + ffi = FFI() + ffi.include(ffi1) + ffi.cdef("mystruct_t *ff6(void);") + lib = verify(ffi, "test_include_6", + "typedef struct _mystruct_s mystruct_t; //usually from a #include\n" + "struct _mystruct_s { int x; };\n" + "static mystruct_t result_struct = { 42 };\n" + "mystruct_t *ff6(void) { return &result_struct; }") + p = lib.ff6() + assert ffi.cast("int *", p)[0] == 42 diff --git a/cffi/cparser.py b/cffi/cparser.py --- a/cffi/cparser.py +++ b/cffi/cparser.py @@ -608,7 +608,7 @@ def include(self, other): for name, tp in other._declarations.items(): kind = name.split(' ', 1)[0] - if kind in ('struct', 'union', 'enum'): + if kind in ('struct', 'union', 'enum', 'anonymous'): self._declare(name, tp, included=True) elif kind == 'typedef': self._declare(name, tp) From noreply at buildbot.pypy.org Tue Apr 28 14:27:39 2015 From: noreply at buildbot.pypy.org (Rob Young) Date: Tue, 28 Apr 2015 14:27:39 +0200 (CEST) Subject: [pypy-commit] cffi default: Skip pkg-config if OSError EACCES is raised Message-ID: <20150428122739.C24C91C14BF@cobra.cs.uni-duesseldorf.de> Author: Rob Young Branch: Changeset: r1873:e1d2dcf322f8 Date: 2015-04-28 12:57 +0100 http://bitbucket.org/cffi/cffi/changeset/e1d2dcf322f8/ Log: Skip pkg-config if OSError EACCES is raised If one of the directories in the PATH is not executable by the current user then EACCES will be raised. diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ p = subprocess.Popen([pkg_config, option, 'libffi'], stdout=subprocess.PIPE) except OSError as e: - if e.errno != errno.ENOENT: + if e.errno not in [errno.ENOENT, errno.EACCES]: raise else: t = p.stdout.read().decode().strip() From noreply at buildbot.pypy.org Tue Apr 28 23:58:27 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Tue, 28 Apr 2015 23:58:27 +0200 (CEST) Subject: [pypy-commit] extradoc extradoc: PyPy for mediocre programmers: the abstract Message-ID: <20150428215827.23A591C04A4@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: extradoc Changeset: r5538:4397b357ffb7 Date: 2015-04-28 22:59 +0100 http://bitbucket.org/pypy/extradoc/changeset/4397b357ffb7/ Log: PyPy for mediocre programmers: the abstract diff --git a/talk/ep2015/pypy-abstract.rst b/talk/ep2015/pypy-abstract.rst new file mode 100644 --- /dev/null +++ b/talk/ep2015/pypy-abstract.rst @@ -0,0 +1,22 @@ +============================= +PyPy for mediocre programmers +============================= + +This is a talk for mediocre Python programmers by a mediocre programmer. PyPy +is an alternative implementation of Python. It is notorious for being fast, but +also for using clever algorithms pertaining to advanced concepts such as type +inference, garbage collection, just-in-time compilation, etc. So, can we, +mediocre programmers, realistically use PyPy? + +Yes, absolutely. In fact, PyPy developers did all that hard work so that we +wouldn't have to. As we'll see, it runs most Python code exactly like CPython +does, save that it magically makes it faster. + +Porting existing applications is always more involved than running a simple +script, so we'll also examine likely difficulties such as code relying on +CPython implementation details, and dependencies on C extensions, and explore +simple principles to let PyPy run your code even faster. + +Finally, we'll have a glimpse of the future by looking at what's brewing in +the PyPy lair, such as software transactional memory, new speed optimisations, +better support for Python 3 and NumPy, ... From noreply at buildbot.pypy.org Wed Apr 29 12:49:15 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 29 Apr 2015 12:49:15 +0200 (CEST) Subject: [pypy-commit] cffi default: Add a note about variadic functions Message-ID: <20150429104915.EFEE81C0EC0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1874:32ec3c43f186 Date: 2015-04-29 12:49 +0200 http://bitbucket.org/cffi/cffi/changeset/32ec3c43f186/ Log: Add a note about variadic functions diff --git a/doc/source/index.rst b/doc/source/index.rst --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -1078,6 +1078,16 @@ C.printf("hello, %f\n", ffi.cast("double", 42)) C.printf("hello, %s\n", ffi.new("char[]", "world")) +Note that if you are using ``dlopen()``, the function declaration in the +``cdef()`` must match the original one in C exactly, as usual --- in +particular, if this function is variadic in C, then its ``cdef()`` +declaration must also be variadic. You cannot declare it in the +``cdef()`` with fixed arguments instead, even if you plan to only call +it with these argument types. The reason is that some architectures +have a different calling convention depending on whether the function +signature is fixed or not. (On x86-64, the difference can sometimes be +seen in PyPy's JIT-generated code if some arguments are ``double``.) + Callbacks --------- From noreply at buildbot.pypy.org Wed Apr 29 13:23:28 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 29 Apr 2015 13:23:28 +0200 (CEST) Subject: [pypy-commit] cffi default: Document what we do with "int foo(); ". Message-ID: <20150429112328.2A0CF1C03F7@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1875:98b0b26c05e2 Date: 2015-04-29 13:24 +0200 http://bitbucket.org/cffi/cffi/changeset/98b0b26c05e2/ Log: Document what we do with "int foo();". diff --git a/doc/source/index.rst b/doc/source/index.rst --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -1088,6 +1088,13 @@ signature is fixed or not. (On x86-64, the difference can sometimes be seen in PyPy's JIT-generated code if some arguments are ``double``.) +Note that the function signature ``int foo();`` is interpreted by CFFI +as equivalent to ``int foo(void);``. This differs from the C standard, +in which ``int foo();`` is really like ``int foo(...);`` and can be +called with any arguments. (This feature of C is a pre-C89 relic: the +arguments cannot be accessed at all in the body of ``foo()`` without +relying on compiler-specific extensions.) + Callbacks --------- From noreply at buildbot.pypy.org Wed Apr 29 13:41:59 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 29 Apr 2015 13:41:59 +0200 (CEST) Subject: [pypy-commit] stmgc default: It is always better to use "void" in the argument list when there is no Message-ID: <20150429114159.1AFAA1C083B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1758:12a01c2778c0 Date: 2015-04-29 13:42 +0200 http://bitbucket.org/pypy/stmgc/changeset/12a01c2778c0/ Log: It is always better to use "void" in the argument list when there is no argument. diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -308,7 +308,7 @@ static void reset_modified_from_backup_copies(int segment_num); /* forward */ -static bool _stm_validate() +static bool _stm_validate(void) { /* returns true if we reached a valid state, or false if we need to abort now */ diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -297,7 +297,7 @@ static void synchronize_objects_flush(void); static void _signal_handler(int sig, siginfo_t *siginfo, void *context); -static bool _stm_validate(); +static bool _stm_validate(void); static inline bool was_read_remote(char *base, object_t *obj) { @@ -329,7 +329,7 @@ spinlock_release(get_priv_segment(segnum)->privatization_lock); } -static inline bool all_privatization_locks_acquired() +static inline bool all_privatization_locks_acquired(void) { #ifndef NDEBUG long l; @@ -343,7 +343,7 @@ #endif } -static inline void acquire_all_privatization_locks() +static inline void acquire_all_privatization_locks(void) { /* XXX: don't do for the sharing seg0 */ long l; @@ -352,7 +352,7 @@ } } -static inline void release_all_privatization_locks() +static inline void release_all_privatization_locks(void) { long l; for (l = NB_SEGMENTS-1; l >= 0; l--) { diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -681,7 +681,7 @@ _stm_smallmalloc_sweep(); } -static void clean_up_commit_log_entries() +static void clean_up_commit_log_entries(void) { struct stm_commit_log_entry_s *cl, *next; From noreply at buildbot.pypy.org Wed Apr 29 13:44:46 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 29 Apr 2015 13:44:46 +0200 (CEST) Subject: [pypy-commit] stmgc default: More of the same Message-ID: <20150429114446.081CC1C083B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1759:64ba29e21134 Date: 2015-04-29 13:45 +0200 http://bitbucket.org/pypy/stmgc/changeset/64ba29e21134/ Log: More of the same diff --git a/c8/stm/misc.c b/c8/stm/misc.c --- a/c8/stm/misc.c +++ b/c8/stm/misc.c @@ -44,7 +44,7 @@ return obj->stm_flags & _STM_GCFLAG_CARDS_SET; } -long _stm_count_cl_entries() +long _stm_count_cl_entries(void) { struct stm_commit_log_entry_s *cl = &commit_log_root; @@ -115,7 +115,7 @@ return cards[get_index_to_card_index(idx)].rm; } -uint8_t _stm_get_transaction_read_version() +uint8_t _stm_get_transaction_read_version(void) { return STM_SEGMENT->transaction_read_version; } @@ -124,7 +124,7 @@ static struct stm_commit_log_entry_s *_last_cl_entry; static long _last_cl_entry_index; -void _stm_start_enum_last_cl_entry() +void _stm_start_enum_last_cl_entry(void) { _last_cl_entry = &commit_log_root; struct stm_commit_log_entry_s *cl = &commit_log_root; @@ -135,7 +135,7 @@ _last_cl_entry_index = 0; } -object_t *_stm_next_last_cl_entry() +object_t *_stm_next_last_cl_entry(void) { if (_last_cl_entry == &commit_log_root) return NULL; @@ -150,7 +150,7 @@ } -void _stm_smallmalloc_sweep_test() +void _stm_smallmalloc_sweep_test(void) { acquire_all_privatization_locks(); _stm_smallmalloc_sweep(); diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -94,13 +94,13 @@ object_t *_stm_allocate_slowpath(ssize_t); object_t *_stm_allocate_external(ssize_t); void _stm_become_inevitable(const char*); -void _stm_collectable_safe_point(); +void _stm_collectable_safe_point(void); object_t *_stm_allocate_old(ssize_t size_rounded_up); char *_stm_real_address(object_t *o); #ifdef STM_TESTS #include -uint8_t _stm_get_transaction_read_version(); +uint8_t _stm_get_transaction_read_version(void); uint8_t _stm_get_card_value(object_t *obj, long idx); bool _stm_was_read(object_t *obj); bool _stm_was_written(object_t *obj); @@ -137,9 +137,9 @@ long _stm_count_objects_pointing_to_nursery(void); object_t *_stm_enum_modified_old_objects(long index); object_t *_stm_enum_objects_pointing_to_nursery(long index); -object_t *_stm_next_last_cl_entry(); -void _stm_start_enum_last_cl_entry(); -long _stm_count_cl_entries(); +object_t *_stm_next_last_cl_entry(void); +void _stm_start_enum_last_cl_entry(void); +long _stm_count_cl_entries(void); long _stm_count_old_objects_with_cards_set(void); object_t *_stm_enum_old_objects_with_cards_set(long index); uint64_t _stm_total_allocated(void); From noreply at buildbot.pypy.org Wed Apr 29 19:29:37 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 29 Apr 2015 19:29:37 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: first attempt at writing setuptools hooks Message-ID: <20150429172937.8D8F51C0823@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1876:35dd6cec69f3 Date: 2015-04-29 19:30 +0200 http://bitbucket.org/cffi/cffi/changeset/35dd6cec69f3/ Log: first attempt at writing setuptools hooks diff --git a/_cffi1/setuptools_ext.py b/_cffi1/setuptools_ext.py new file mode 100644 --- /dev/null +++ b/_cffi1/setuptools_ext.py @@ -0,0 +1,80 @@ + +def error(msg): + from distutils.errors import DistutilsSetupError + raise DistutilsSetupError(msg) + + +def add_cffi_module(dist, mod_spec): + import os + from cffi.api import FFI + from _cffi1 import recompiler + from distutils.core import Extension + from distutils.command.build_ext import build_ext + from distutils.dir_util import mkpath + from distutils import log + + if not isinstance(mod_spec, str): + error("argument to 'cffi_modules=...' must be a str or a list of str," + " not %r" % (type(mod_spec).__name__,)) + try: + build_mod_name, ffi_var_name = mod_spec.split(':') + except ValueError: + error("%r must be of the form 'build_mod_name:ffi_variable'" % + (mod_spec,)) + mod = __import__(build_mod_name, None, None, [ffi_var_name]) + try: + ffi = getattr(mod, ffi_var_name) + except AttributeError: + error("%r: object %r not found in module" % (mod_spec, + ffi_var_name)) + if not isinstance(ffi, FFI): + error("%r is not an FFI instance (got %r)" % (mod_spec, + type(ffi).__name__)) + if not hasattr(ffi, '_assigned_source'): + error("%r: the set_source() method was not called" % (mod_spec,)) + module_name = ffi._recompiler_module_name + source, kwds = ffi._assigned_source + + allsources = ['$PLACEHOLDER'] + allsources.extend(kwds.get('sources', [])) + ext = Extension(name=module_name, sources=allsources, **kwds) + + def make_mod(tmpdir): + mkpath(tmpdir) + file_name = module_name + '.c' + log.info("generating cffi module %r" % file_name) + c_file = os.path.join(tmpdir, file_name) + c_tmp = '%s.%s' % (c_file, os.getpid()) + recompiler.make_c_source(ffi, module_name, source, c_tmp) + try: + with open(c_file, 'r') as f1: + with open(c_tmp, 'r') as f2: + if f1.read() != f2.read(): + raise IOError + except IOError: + os.rename(c_tmp, c_file) + else: + log.info("already up-to-date") + os.unlink(c_tmp) + return c_file + + if dist.ext_modules is None: + dist.ext_modules = [] + dist.ext_modules.append(ext) + + base_class = dist.cmdclass.get('build_ext', build_ext) + class build_ext_make_mod(base_class): + def run(self): + if ext.sources[0] == '$PLACEHOLDER': + ext.sources[0] = make_mod(self.build_temp) + base_class.run(self) + dist.cmdclass['build_ext'] = build_ext_make_mod + + +def cffi_modules(dist, attr, value): + assert attr == 'cffi_modules' + if isinstance(value, str): + value = [value] + + for cffi_module in value: + add_cffi_module(dist, cffi_module) diff --git a/cffi/api.py b/cffi/api.py --- a/cffi/api.py +++ b/cffi/api.py @@ -475,6 +475,22 @@ ('_UNICODE', '1')] kwds['define_macros'] = defmacros + def set_source(self, module_name, source, **kwds): + if hasattr(self, '_assigned_source'): + raise ValueError("set_source() cannot be called several times " + "per ffi object") + self._recompiler_module_name = module_name + self._assigned_source = (source, kwds) + + def compile(self, tmpdir='.'): + from _cffi1 import recompile + # + if not hasattr(self, '_assigned_source'): + raise ValueError("set_source() must be called before compile()") + source, kwds = self._assigned_source + return recompile(self, self._recompiler_module_name, + source, tmpdir=tmpdir, **kwds) + def _load_backend_lib(backend, name, flags): if name is None: diff --git a/demo/bsdopendirtype_build.py b/demo/bsdopendirtype_build.py --- a/demo/bsdopendirtype_build.py +++ b/demo/bsdopendirtype_build.py @@ -1,5 +1,4 @@ from cffi import FFI -from _cffi1 import recompile ffi = FFI() ffi.cdef(""" @@ -15,7 +14,10 @@ static const int DT_BLK, DT_CHR, DT_DIR, DT_FIFO, DT_LNK, DT_REG, DT_SOCK; """) -recompile(ffi, "_bsdopendirtype", """ +ffi.set_source("_bsdopendirtype", """ #include #include """) + +if __name__ == '__main__': + ffi.compile() diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -142,8 +142,9 @@ `Mailing list `_ """, - version='0.9.2', - packages=['cffi'], + version='1.0.0', + packages=['cffi', '_cffi1'], + package_data={'_cffi1': ['_cffi_include.h', 'parse_c_type.h']}, zip_safe=False, url='http://cffi.readthedocs.org', @@ -157,6 +158,13 @@ install_requires=[ 'pycparser', ], + + entry_points = { + "distutils.setup_keywords": [ + "cffi_modules = _cffi1.setuptools_ext:cffi_modules", + ], + }, + classifiers=[ 'Programming Language :: Python', 'Programming Language :: Python :: 2', From noreply at buildbot.pypy.org Wed Apr 29 19:41:14 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 29 Apr 2015 19:41:14 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Add the setup.py corresponding to the demo Message-ID: <20150429174114.DDC7C1C03F7@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1877:d0455ef2bbb2 Date: 2015-04-29 19:41 +0200 http://bitbucket.org/cffi/cffi/changeset/d0455ef2bbb2/ Log: Add the setup.py corresponding to the demo diff --git a/demo/bsdopendirtype_setup.py b/demo/bsdopendirtype_setup.py --- a/demo/bsdopendirtype_setup.py +++ b/demo/bsdopendirtype_setup.py @@ -1,22 +1,13 @@ -from cffi import FFI +from setuptools import setup -ffi = FFI() -ffi.csource("_bsdopendirtype", """ - #include - #include -""") -ffi.cdef(""" - typedef ... DIR; - struct dirent { - unsigned char d_type; /* type of file */ - char d_name[]; /* filename */ - ...; - }; - DIR *opendir(const char *name); - int closedir(DIR *dirp); - struct dirent *readdir(DIR *dirp); - static const int DT_BLK, DT_CHR, DT_DIR, DT_FIFO, DT_LNK, DT_REG, DT_SOCK; -""") - -if __name__ == '__main__': - ffi.recompile() +setup( + name="example", + version="0.1", + py_modules=["bsdopendirtype"], + setup_requires=["cffi>=1.0"], + cffi_modules=[ + "bsdopendirtype_build:ffi", + ], + install_requires=["cffi>=1.0"], # should maybe be "cffi-backend" only? + zip_safe=False, +) From noreply at buildbot.pypy.org Wed Apr 29 20:02:37 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 29 Apr 2015 20:02:37 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Change the other two demos to the ffi.set_source() style too Message-ID: <20150429180237.C4D961C0823@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1878:68b878e3ccbb Date: 2015-04-29 20:03 +0200 http://bitbucket.org/cffi/cffi/changeset/68b878e3ccbb/ Log: Change the other two demos to the ffi.set_source() style too diff --git a/demo/_curses_build.py b/demo/_curses_build.py --- a/demo/_curses_build.py +++ b/demo/_curses_build.py @@ -4,7 +4,6 @@ raise ImportError('No module named _curses') from cffi import FFI -from _cffi1 import recompile ffi = FFI() @@ -283,7 +282,7 @@ """) -recompile(ffi, "_curses_cffi", """ +ffi.set_source("_curses_cffi", """ #ifdef __APPLE__ /* the following define is necessary for OS X 10.6+; without it, the Apple-supplied ncurses.h sets NCURSES_OPAQUE to 1, and then Python @@ -323,3 +322,6 @@ getsyx(yx[0], yx[1]); } """, libraries=['ncurses', 'panel']) + +if __name__ == '__main__': + ffi.compile() diff --git a/demo/_curses_setup.py b/demo/_curses_setup.py new file mode 100644 --- /dev/null +++ b/demo/_curses_setup.py @@ -0,0 +1,13 @@ +from setuptools import setup + +setup( + name="_curses", + version="0.1", + py_modules=["_curses"], + setup_requires=["cffi>=1.0"], + cffi_modules=[ + "_curses_build:ffi", + ], + install_requires=["cffi>=1.0"], # should maybe be "cffi-backend" only? + zip_safe=False, +) diff --git a/demo/readdir2_build.py b/demo/readdir2_build.py --- a/demo/readdir2_build.py +++ b/demo/readdir2_build.py @@ -1,5 +1,4 @@ from cffi import FFI -from _cffi1 import recompile ffi = FFI() ffi.cdef(""" @@ -21,7 +20,7 @@ static const int DT_DIR; """) -recompile(ffi, "_readdir2", """ +ffi.set_source("_readdir2", """ #ifndef _ATFILE_SOURCE # define _ATFILE_SOURCE #endif @@ -32,3 +31,6 @@ #include #include """) + +if __name__ == '__main__': + ffi.compile() diff --git a/demo/readdir2_setup.py b/demo/readdir2_setup.py new file mode 100644 --- /dev/null +++ b/demo/readdir2_setup.py @@ -0,0 +1,13 @@ +from setuptools import setup + +setup( + name="readdir2", + version="0.1", + py_modules=["readdir2"], + setup_requires=["cffi>=1.0"], + cffi_modules=[ + "readdir2_build:ffi", + ], + install_requires=["cffi>=1.0"], # should maybe be "cffi-backend" only? + zip_safe=False, +) From noreply at buildbot.pypy.org Wed Apr 29 20:13:51 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 29 Apr 2015 20:13:51 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: A demo using ffi.include() --- which doesn't work right now Message-ID: <20150429181351.7A8B41C0207@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1879:28be0615f2f3 Date: 2015-04-29 20:14 +0200 http://bitbucket.org/cffi/cffi/changeset/28be0615f2f3/ Log: A demo using ffi.include() --- which doesn't work right now diff --git a/demo/recopendirtype.py b/demo/recopendirtype.py new file mode 100644 --- /dev/null +++ b/demo/recopendirtype.py @@ -0,0 +1,50 @@ +from _recopendirtype import ffi, lib + + +def _posix_error(): + raise OSError(ffi.errno, os.strerror(ffi.errno)) + +_dtype_to_smode = { + lib.DT_BLK: 0o060000, + lib.DT_CHR: 0o020000, + lib.DT_DIR: 0o040000, + lib.DT_FIFO: 0o010000, + lib.DT_LNK: 0o120000, + lib.DT_REG: 0o100000, + lib.DT_SOCK: 0o140000, +} + +def opendir(dir): + if len(dir) == 0: + dir = '.' + dirname = dir + if not dirname.endswith('/'): + dirname += '/' + dirp = lib.opendir(dir) + if dirp == ffi.NULL: + raise _posix_error() + dirent = ffi.new("struct dirent *") + result = ffi.new("struct dirent **") + try: + while True: + ffi.errno = 0 + err = lib.readdir_r(dirp, dirent, result) + if err: # really got an error + raise OSError(err, os.strerror(err)) + if result[0] == ffi.NULL: + return # + name = ffi.string(dirent.d_name) + if name == '.' or name == '..': + continue + name = dirname + name + try: + smode = _dtype_to_smode[dirent.d_type] + except KeyError: + smode = os.lstat(name).st_mode + yield name, smode + finally: + lib.closedir(dirp) + +if __name__ == '__main__': + for name, smode in opendir('/tmp'): + print hex(smode), name diff --git a/demo/recopendirtype_build.py b/demo/recopendirtype_build.py new file mode 100644 --- /dev/null +++ b/demo/recopendirtype_build.py @@ -0,0 +1,19 @@ +from cffi import FFI +import bsdopendirtype_build + +ffi = FFI() + +# ========== This is a demo of ffi.include() ========== +ffi.include(bsdopendirtype_build.ffi) + +ffi.cdef(""" + int readdir_r(DIR *dirp, struct dirent *entry, struct dirent **result); +""") + +ffi.set_source("_recopendirtype", """ + #include + #include +""") + +if __name__ == '__main__': + ffi.compile() From noreply at buildbot.pypy.org Wed Apr 29 20:22:46 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 29 Apr 2015 20:22:46 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Test and fix Message-ID: <20150429182246.0F3131C0207@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1880:e5f48577bca9 Date: 2015-04-29 20:23 +0200 http://bitbucket.org/cffi/cffi/changeset/e5f48577bca9/ Log: Test and fix diff --git a/_cffi1/test_recompiler.py b/_cffi1/test_recompiler.py --- a/_cffi1/test_recompiler.py +++ b/_cffi1/test_recompiler.py @@ -568,11 +568,31 @@ "typedef struct _mystruct_s mystruct_t;") ffi = FFI() ffi.include(ffi1) - ffi.cdef("mystruct_t *ff6(void);") + ffi.cdef("mystruct_t *ff6(void); int ff6b(mystruct_t *);") lib = verify(ffi, "test_include_6", "typedef struct _mystruct_s mystruct_t; //usually from a #include\n" "struct _mystruct_s { int x; };\n" "static mystruct_t result_struct = { 42 };\n" - "mystruct_t *ff6(void) { return &result_struct; }") + "mystruct_t *ff6(void) { return &result_struct; }\n" + "int ff6b(mystruct_t *p) { return p->x; }") p = lib.ff6() assert ffi.cast("int *", p)[0] == 42 + assert lib.ff6b(p) == 42 + +def test_include_7(): + ffi1 = FFI() + ffi1.cdef("typedef ... mystruct_t;\n" + "int ff7b(mystruct_t *);") + verify(ffi1, "test_include_7_parent", + "typedef struct { int x; } mystruct_t;\n" + "int ff7b(mystruct_t *p) { return p->x; }") + ffi = FFI() + ffi.include(ffi1) + ffi.cdef("mystruct_t *ff7(void);") + lib = verify(ffi, "test_include_7", + "typedef struct { int x; } mystruct_t; //usually from a #include\n" + "static mystruct_t result_struct = { 42 };" + "mystruct_t *ff7(void) { return &result_struct; }") + p = lib.ff7() + assert ffi.cast("int *", p)[0] == 42 + assert lib.ff7b(p) == 42 diff --git a/cffi/cparser.py b/cffi/cparser.py --- a/cffi/cparser.py +++ b/cffi/cparser.py @@ -611,6 +611,6 @@ if kind in ('struct', 'union', 'enum', 'anonymous'): self._declare(name, tp, included=True) elif kind == 'typedef': - self._declare(name, tp) + self._declare(name, tp, included=True) for k, v in other._int_constants.items(): self._add_constants(k, v) From noreply at buildbot.pypy.org Wed Apr 29 21:28:27 2015 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 29 Apr 2015 21:28:27 +0200 (CEST) Subject: [pypy-commit] pypy numpy-fixes: add implementation of 9ea844c184eb Message-ID: <20150429192827.531241C068C@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: numpy-fixes Changeset: r76947:43445ac2ff6b Date: 2015-04-27 20:27 +0300 http://bitbucket.org/pypy/pypy/changeset/43445ac2ff6b/ Log: add implementation of 9ea844c184eb diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -1305,12 +1305,30 @@ @complex_binary_op def floordiv(self, v1, v2): - try: - ab = v1[0]*v2[0] + v1[1]*v2[1] - bb = v2[0]*v2[0] + v2[1]*v2[1] - return math.floor(ab/bb), 0. - except ZeroDivisionError: - return rfloat.NAN, 0. + (r1, i1), (r2, i2) = v1, v2 + if r2 < 0: + abs_r2 = -r2 + else: + abs_r2 = r2 + if i2 < 0: + abs_i2 = -i2 + else: + abs_i2 = i2 + if abs_r2 >= abs_i2: + if abs_r2 == 0.0: + return rfloat.NAN, 0. + else: + ratio = i2 / r2 + denom = r2 + i2 * ratio + rr = (r1 + i1 * ratio) / denom + elif rfloat.isnan(r2): + rr = rfloat.NAN + else: + ratio = r2 / i2 + denom = r2 * ratio + i2 + assert i2 != 0.0 + rr = (r1 * ratio + i1) / denom + return math.floor(rr), 0. #complex mod does not exist in numpy #@simple_binary_op From noreply at buildbot.pypy.org Wed Apr 29 21:28:28 2015 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 29 Apr 2015 21:28:28 +0200 (CEST) Subject: [pypy-commit] pypy numpy-fixes: special case sign(nan) Message-ID: <20150429192828.90AD91C068C@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: numpy-fixes Changeset: r76948:15dee07ee651 Date: 2015-04-28 00:18 +0300 http://bitbucket.org/pypy/pypy/changeset/15dee07ee651/ Log: special case sign(nan) diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -722,6 +722,8 @@ def sign(self, v): if v == 0.0: return 0.0 + if rfloat.isnan(v): + return rfloat.NAN return rfloat.copysign(1.0, v) @raw_unary_op From noreply at buildbot.pypy.org Wed Apr 29 21:28:29 2015 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 29 Apr 2015 21:28:29 +0200 (CEST) Subject: [pypy-commit] pypy numpy-fixes: make view actually a view rather than a copy if dtype is same-length Message-ID: <20150429192829.D59681C068C@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: numpy-fixes Changeset: r76949:8e4a51fc2b92 Date: 2015-04-28 22:53 +0300 http://bitbucket.org/pypy/pypy/changeset/8e4a51fc2b92/ Log: make view actually a view rather than a copy if dtype is same-length diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py --- a/pypy/module/micronumpy/concrete.py +++ b/pypy/module/micronumpy/concrete.py @@ -90,9 +90,13 @@ new_shape, self, orig_array) return None - def get_view(self, space, orig_array, dtype, new_shape): - strides, backstrides = calc_strides(new_shape, dtype, + def get_view(self, space, orig_array, dtype, new_shape, reuse_strides=False): + if not reuse_strides: + strides, backstrides = calc_strides(new_shape, dtype, self.order) + else: + strides = self.get_strides() + backstrides = self.get_backstrides() return SliceArray(self.start, strides, backstrides, new_shape, self, orig_array, dtype=dtype) @@ -328,11 +332,7 @@ return ArrayBuffer(self, readonly) def astype(self, space, dtype): - # we want to create a new array, but must respect the strides - # in self. So find a factor of the itemtype.elsize, and use this - factor = float(dtype.elsize) / self.dtype.elsize - strides = [int(factor*s) for s in self.get_strides()] - backstrides = [int(factor*s) for s in self.get_backstrides()] + strides, backstrides = calc_strides(self.get_shape(), dtype, self.order) impl = ConcreteArray(self.get_shape(), dtype, self.order, strides, backstrides) loop.setslice(space, impl.get_shape(), impl, self) diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -850,7 +850,15 @@ raise OperationError(space.w_ValueError, space.wrap( "new type not compatible with array.")) # Strides, shape does not change - v = impl.astype(space, dtype) + if dtype.is_object() != impl.dtype.is_object(): + raise oefmt(space.w_ValueError, 'expect trouble in ndarray.view,' + ' target dtype %r but self.dtype %r',dtype, impl.dtype) + + base = impl.base() + if base is None: + base = self + v = impl.get_view(space, base, dtype, self.get_shape(), + reuse_strides=True) return wrap_impl(space, w_type, self, v) strides = impl.get_strides() if dims == 1 or strides[0] =0 - assert offset < storage._obj.getlength() + try: + assert offset < storage._obj.getlength() + except AttributeError: + pass return _raw_storage_setitem_unaligned(storage, offset, value) def raw_storage_getitem_unaligned(T, storage, offset): assert offset >=0 - assert offset < storage._obj.getlength() + try: + assert offset < storage._obj.getlength() + except AttributeError: + pass return _raw_storage_getitem_unaligned(T, storage, offset) ''' - def simple_unary_op(func): specialize.argtype(1)(func) @functools.wraps(func) From noreply at buildbot.pypy.org Wed Apr 29 21:28:31 2015 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 29 Apr 2015 21:28:31 +0200 (CEST) Subject: [pypy-commit] pypy numpy-fixes: typos, add failing view test Message-ID: <20150429192831.0EEC91C068C@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: numpy-fixes Changeset: r76950:f169593652bc Date: 2015-04-29 08:12 +0300 http://bitbucket.org/pypy/pypy/changeset/f169593652bc/ Log: typos, add failing view test diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -852,7 +852,7 @@ # Strides, shape does not change if dtype.is_object() != impl.dtype.is_object(): raise oefmt(space.w_ValueError, 'expect trouble in ndarray.view,' - ' target dtype %r but self.dtype %r',dtype, impl.dtype) + ' one of target dtype or dtype is object dtype') base = impl.base() if base is None: diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -1820,6 +1820,10 @@ assert v.strides == s.strides assert v.base is s.base assert (v == 2).all() + y = empty([6,6], 'uint32') + s = y.swapaxes(0, 1) + v = s.view(y.__class__) + assert v.strides == (4, 24) def test_tolist_scalar(self): from numpy import dtype diff --git a/pypy/module/micronumpy/test/test_subtype.py b/pypy/module/micronumpy/test/test_subtype.py --- a/pypy/module/micronumpy/test/test_subtype.py +++ b/pypy/module/micronumpy/test/test_subtype.py @@ -405,7 +405,7 @@ def test_setstate_no_version(self): # Some subclasses of ndarray, like MaskedArray, do not use - # version in __setstare__ + # version in __setstate__ from numpy import ndarray, array from pickle import loads, dumps import sys, new From noreply at buildbot.pypy.org Wed Apr 29 21:28:32 2015 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 29 Apr 2015 21:28:32 +0200 (CEST) Subject: [pypy-commit] pypy numpy-fixes: simplify and fix logic of strides, shapes for view() Message-ID: <20150429192832.30F231C068C@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: numpy-fixes Changeset: r76951:ad108273e79b Date: 2015-04-29 22:28 +0300 http://bitbucket.org/pypy/pypy/changeset/ad108273e79b/ Log: simplify and fix logic of strides, shapes for view() diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py --- a/pypy/module/micronumpy/concrete.py +++ b/pypy/module/micronumpy/concrete.py @@ -90,13 +90,10 @@ new_shape, self, orig_array) return None - def get_view(self, space, orig_array, dtype, new_shape, reuse_strides=False): - if not reuse_strides: + def get_view(self, space, orig_array, dtype, new_shape, strides=None, backstrides=None): + if not strides: strides, backstrides = calc_strides(new_shape, dtype, self.order) - else: - strides = self.get_strides() - backstrides = self.get_backstrides() return SliceArray(self.start, strides, backstrides, new_shape, self, orig_array, dtype=dtype) diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -844,36 +844,35 @@ if old_itemsize != new_itemsize: raise OperationError(space.w_ValueError, space.wrap( "new type not compatible with array.")) + strides = None + backstrides = None + base = self else: - if not is_c_contiguous(impl) and not is_f_contiguous(impl): - if old_itemsize != new_itemsize: + base = impl.base() + if base is None: + base = self + strides = impl.get_strides()[:] + backstrides = impl.get_backstrides()[:] + if old_itemsize != new_itemsize: + if not is_c_contiguous(impl) and not is_f_contiguous(impl): raise OperationError(space.w_ValueError, space.wrap( "new type not compatible with array.")) - # Strides, shape does not change - if dtype.is_object() != impl.dtype.is_object(): - raise oefmt(space.w_ValueError, 'expect trouble in ndarray.view,' - ' one of target dtype or dtype is object dtype') - - base = impl.base() - if base is None: - base = self - v = impl.get_view(space, base, dtype, self.get_shape(), - reuse_strides=True) - return wrap_impl(space, w_type, self, v) - strides = impl.get_strides() - if dims == 1 or strides[0] Author: Armin Rigo Branch: cffi-1.0 Changeset: r1881:a41b31913b62 Date: 2015-04-29 22:22 +0200 http://bitbucket.org/cffi/cffi/changeset/a41b31913b62/ Log: Python 3 compat diff --git a/_cffi1/__init__.py b/_cffi1/__init__.py --- a/_cffi1/__init__.py +++ b/_cffi1/__init__.py @@ -1,1 +1,1 @@ -from recompiler import make_c_source, recompile +from .recompiler import make_c_source, recompile diff --git a/_cffi1/_cffi_include.h b/_cffi1/_cffi_include.h --- a/_cffi1/_cffi_include.h +++ b/_cffi1/_cffi_include.h @@ -148,7 +148,8 @@ #define _cffi_convert_array_from_object \ ((int(*)(char *, CTypeDescrObject *, PyObject *))_cffi_exports[24]) #define _cffi_init_module \ - ((int(*)(char *, const struct _cffi_type_context_s *))_cffi_exports[25]) + ((PyObject *(*)(char *, const struct _cffi_type_context_s *)) \ + _cffi_exports[25]) #define _CFFI_NUM_EXPORTS 26 typedef struct _ctypedescr CTypeDescrObject; diff --git a/_cffi1/cffi1_module.c b/_cffi1/cffi1_module.c --- a/_cffi1/cffi1_module.c +++ b/_cffi1/cffi1_module.c @@ -98,25 +98,46 @@ return -1; } -static int _cffi_init_module(char *module_name, - const struct _cffi_type_context_s *ctx) +static PyObject *_cffi_init_module(char *module_name, + const struct _cffi_type_context_s *ctx) { - PyObject *m = Py_InitModule(module_name, NULL); + PyObject *m; + +#if PY_MAJOR_VERSION >= 3 + /* note: the module_def leaks, but anyway the C extension module cannot + be unloaded */ + struct PyModuleDef *module_def; + module_def = PyObject_Malloc(sizeof(struct PyModuleDef)); + if (module_def == NULL) + return PyErr_NoMemory(); + + struct PyModuleDef local_module_def = { + PyModuleDef_HEAD_INIT, + module_name, + NULL, + -1, + NULL, NULL, NULL, NULL, NULL + }; + *module_def = local_module_def; + m = PyModule_Create(module_def); +#else + m = Py_InitModule(module_name, NULL); +#endif if (m == NULL) - return -1; + return NULL; FFIObject *ffi = ffi_internal_new(&FFI_Type, ctx); Py_XINCREF(ffi); /* make the ffi object really immortal */ if (ffi == NULL || PyModule_AddObject(m, "ffi", (PyObject *)ffi) < 0) - return -1; + return NULL; LibObject *lib = lib_internal_new(ffi->types_builder, module_name); if (lib == NULL || PyModule_AddObject(m, "lib", (PyObject *)lib) < 0) - return -1; + return NULL; if (make_included_tuples(ctx->includes, &ffi->types_builder->included_ffis, &lib->l_includes) < 0) - return -1; + return NULL; - return 0; + return m; } diff --git a/_cffi1/cffi_opcode.py b/_cffi1/cffi_opcode.py --- a/_cffi1/cffi_opcode.py +++ b/_cffi1/cffi_opcode.py @@ -139,6 +139,6 @@ F_EXTERNAL = 0x08 CLASS_NAME = {} -for _name, _value in globals().items(): +for _name, _value in list(globals().items()): if _name.startswith('OP_') and isinstance(_value, int): CLASS_NAME[_value] = _name[3:] diff --git a/_cffi1/cgc.c b/_cffi1/cgc.c --- a/_cffi1/cgc.c +++ b/_cffi1/cgc.c @@ -39,7 +39,7 @@ PyObject *data; if (const_name_pop == NULL) { - const_name_pop = PyString_InternFromString("pop"); + const_name_pop = PyText_InternFromString("pop"); if (const_name_pop == NULL) return NULL; } diff --git a/_cffi1/ffi_obj.c b/_cffi1/ffi_obj.c --- a/_cffi1/ffi_obj.c +++ b/_cffi1/ffi_obj.c @@ -424,11 +424,11 @@ char *p; base_name_len = strlen(ct->ct_name); - result = PyString_FromStringAndSize(NULL, base_name_len + extra_text_len); + result = PyBytes_FromStringAndSize(NULL, base_name_len + extra_text_len); if (result == NULL) return NULL; - p = PyString_AS_STRING(result); + p = PyBytes_AS_STRING(result); memcpy(p, ct->ct_name, ct->ct_name_position); p += ct->ct_name_position; p += extra_text_len; @@ -473,7 +473,7 @@ if (res == NULL) return NULL; - p = PyString_AS_STRING(res) + ct->ct_name_position; + p = PyBytes_AS_STRING(res) + ct->ct_name_position; if (add_paren) *p++ = '('; if (add_space) @@ -481,6 +481,16 @@ memcpy(p, replace_with, replace_with_len); if (add_paren) p[replace_with_len] = ')'; + +#if PY_MAJOR_VERSION >= 3 + /* bytes -> unicode string */ + PyObject *u = PyUnicode_DecodeLatin1(PyBytes_AS_STRING(res), + PyBytes_GET_SIZE(res), + NULL); + Py_DECREF(res); + res = u; +#endif + return res; } diff --git a/_cffi1/lib_obj.c b/_cffi1/lib_obj.c --- a/_cffi1/lib_obj.c +++ b/_cffi1/lib_obj.c @@ -311,7 +311,7 @@ int i; for (i = 0; i < total; i++) { - PyObject *s = PyString_FromString(g[i].name); + PyObject *s = PyText_FromString(g[i].name); if (s == NULL) { Py_DECREF(lst); return NULL; @@ -370,7 +370,7 @@ LibObject *lib; PyObject *libname, *dict; - libname = PyString_FromString(module_name); + libname = PyText_FromString(module_name); dict = PyDict_New(); if (libname == NULL || dict == NULL) { Py_XDECREF(dict); diff --git a/_cffi1/realize_c_type.c b/_cffi1/realize_c_type.c --- a/_cffi1/realize_c_type.c +++ b/_cffi1/realize_c_type.c @@ -474,7 +474,7 @@ j = 0; while (p[j] != ',' && p[j] != '\0') j++; - tmp = PyString_FromStringAndSize(p, j); + tmp = PyText_FromStringAndSize(p, j); if (tmp == NULL) break; PyTuple_SET_ITEM(enumerators, i, tmp); diff --git a/_cffi1/recompiler.py b/_cffi1/recompiler.py --- a/_cffi1/recompiler.py +++ b/_cffi1/recompiler.py @@ -1,6 +1,6 @@ import os, sys from cffi import ffiplatform, model -from cffi_opcode import * +from .cffi_opcode import * class Recompiler: @@ -225,6 +225,16 @@ # # the init function, loading _cffi_backend and calling a method there base_module_name = self.module_name.split('.')[-1] + prnt('#if PY_MAJOR_VERSION >= 3') + prnt('PyMODINIT_FUNC') + prnt('PyInit_%s(void)' % (base_module_name,)) + prnt('{') + prnt(' if (_cffi_init() < 0)') + prnt(' return NULL;') + prnt(' return _cffi_init_module("%s", &_cffi_type_context);' % ( + self.module_name,)) + prnt('}') + prnt('#else') prnt('PyMODINIT_FUNC') prnt('init%s(void)' % (base_module_name,)) prnt('{') @@ -233,6 +243,7 @@ prnt(' _cffi_init_module("%s", &_cffi_type_context);' % ( self.module_name,)) prnt('}') + prnt('#endif') self.ffi._recompiler_module_name = self.module_name # ---------- diff --git a/_cffi1/support.py b/_cffi1/support.py --- a/_cffi1/support.py +++ b/_cffi1/support.py @@ -1,6 +1,19 @@ +import sys -class U(object): - def __add__(self, other): - return eval('u'+repr(other).replace(r'\\u', r'\u') - .replace(r'\\U', r'\U')) -u = U() +if sys.version_info < (3,): + __all__ = ['u'] + + class U(object): + def __add__(self, other): + return eval('u'+repr(other).replace(r'\\u', r'\u') + .replace(r'\\U', r'\U')) + u = U() + assert u+'a\x00b' == eval(r"u'a\x00b'") + assert u+'a\u1234b' == eval(r"u'a\u1234b'") + assert u+'a\U00012345b' == eval(r"u'a\U00012345b'") + +else: + __all__ = ['u', 'unicode', 'long'] + u = "" + unicode = str + long = int diff --git a/_cffi1/test_ffi_obj.py b/_cffi1/test_ffi_obj.py --- a/_cffi1/test_ffi_obj.py +++ b/_cffi1/test_ffi_obj.py @@ -57,8 +57,8 @@ def test_ffi_string(): ffi = _cffi1_backend.FFI() - p = ffi.new("char[]", "foobar\x00baz") - assert ffi.string(p) == "foobar" + p = ffi.new("char[]", b"foobar\x00baz") + assert ffi.string(p) == b"foobar" def test_ffi_errno(): # xxx not really checking errno, just checking that we can read/write it diff --git a/_cffi1/test_new_ffi_1.py b/_cffi1/test_new_ffi_1.py --- a/_cffi1/test_new_ffi_1.py +++ b/_cffi1/test_new_ffi_1.py @@ -1513,8 +1513,8 @@ outputfilename) ffi2 = module.ffi # - p = ffi2.new("struct nonpacked *", ['A', -43141]) - assert p.a == 'A' + p = ffi2.new("struct nonpacked *", [b'A', -43141]) + assert p.a == b'A' assert p.b == -43141 # p = ffi.new("union simple_u *", [-52525]) diff --git a/_cffi1/test_parse_c_type.py b/_cffi1/test_parse_c_type.py --- a/_cffi1/test_parse_c_type.py +++ b/_cffi1/test_parse_c_type.py @@ -32,7 +32,7 @@ assert global_names == sorted(global_names) ctx = ffi.new("struct _cffi_type_context_s *") -c_struct_names = [ffi.new("char[]", _n) for _n in struct_names] +c_struct_names = [ffi.new("char[]", _n.encode('ascii')) for _n in struct_names] ctx_structs = ffi.new("struct _cffi_struct_union_s[]", len(struct_names)) for _i in range(len(struct_names)): ctx_structs[_i].name = c_struct_names[_i] @@ -40,14 +40,15 @@ ctx.struct_unions = ctx_structs ctx.num_struct_unions = len(struct_names) -c_enum_names = [ffi.new("char[]", _n) for _n in enum_names] +c_enum_names = [ffi.new("char[]", _n.encode('ascii')) for _n in enum_names] ctx_enums = ffi.new("struct _cffi_enum_s[]", len(enum_names)) for _i in range(len(enum_names)): ctx_enums[_i].name = c_enum_names[_i] ctx.enums = ctx_enums ctx.num_enums = len(enum_names) -c_identifier_names = [ffi.new("char[]", _n) for _n in identifier_names] +c_identifier_names = [ffi.new("char[]", _n.encode('ascii')) + for _n in identifier_names] ctx_identifiers = ffi.new("struct _cffi_typename_s[]", len(identifier_names)) for _i in range(len(identifier_names)): ctx_identifiers[_i].name = c_identifier_names[_i] @@ -69,7 +70,7 @@ return 1 ctx_globals = ffi.new("struct _cffi_global_s[]", len(global_names)) -c_glob_names = [ffi.new("char[]", _n) for _n in global_names] +c_glob_names = [ffi.new("char[]", _n.encode('ascii')) for _n in global_names] for _i, _fn in enumerate([fetch_constant_five, fetch_constant_neg, fetch_constant_zero]): @@ -90,9 +91,9 @@ info.output_size = len(out) for j in range(len(out)): out[j] = ffi.cast("void *", -424242) - res = lib.parse_c_type(info, input) + res = lib.parse_c_type(info, input.encode('ascii')) if res < 0: - raise ParseError(ffi.string(info.error_message), + raise ParseError(ffi.string(info.error_message).decode('ascii'), info.error_location) assert 0 <= res < len(out) result = [] diff --git a/_cffi1/test_realize_c_type.py b/_cffi1/test_realize_c_type.py --- a/_cffi1/test_realize_c_type.py +++ b/_cffi1/test_realize_c_type.py @@ -43,6 +43,6 @@ check("int(*)(long[5])", "int(*)(long *)") def test_all_primitives(): - import cffi_opcode + from . import cffi_opcode for name in cffi_opcode.PRIMITIVE_TO_INDEX: check(name, name) diff --git a/_cffi1/test_recompiler.py b/_cffi1/test_recompiler.py --- a/_cffi1/test_recompiler.py +++ b/_cffi1/test_recompiler.py @@ -310,7 +310,7 @@ ffi = FFI() ffi.cdef("""enum e1 { B1, A1, ... }; enum e2 { B2, A2, ... };""") lib = verify(ffi, 'test_verify_enum', - "enum e1 { A1, B1, C1=%d };" % sys.maxint + + "enum e1 { A1, B1, C1=%d };" % sys.maxsize + "enum e2 { A2, B2, C2 };") ffi.typeof("enum e1") ffi.typeof("enum e2") @@ -407,10 +407,10 @@ assert repr(ffi.cast("e1", 2)) == "" # ffi = FFI() - ffi.cdef("typedef enum { AA=%d } e1;" % sys.maxint) + ffi.cdef("typedef enum { AA=%d } e1;" % sys.maxsize) lib = verify(ffi, 'test_verify_anonymous_enum_with_typedef2', - "typedef enum { AA=%d } e1;" % sys.maxint) - assert lib.AA == sys.maxint + "typedef enum { AA=%d } e1;" % sys.maxsize) + assert lib.AA == sys.maxsize assert ffi.sizeof("e1") == ffi.sizeof("long") def test_unique_types(): diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -72,6 +72,7 @@ # define PyText_FromString PyUnicode_FromString # define PyText_FromStringAndSize PyUnicode_FromStringAndSize # define PyText_InternInPlace PyUnicode_InternInPlace +# define PyText_InternFromString PyUnicode_InternFromString # define PyIntOrLong_Check PyLong_Check #else # define STR_OR_BYTES "str" @@ -85,6 +86,7 @@ # define PyText_FromString PyString_FromString # define PyText_FromStringAndSize PyString_FromStringAndSize # define PyText_InternInPlace PyString_InternInPlace +# define PyText_InternFromString PyString_InternFromString # define PyIntOrLong_Check(op) (PyInt_Check(op) || PyLong_Check(op)) #endif @@ -92,6 +94,7 @@ # define PyInt_FromLong PyLong_FromLong # define PyInt_FromSsize_t PyLong_FromSsize_t # define PyInt_AsSsize_t PyLong_AsSsize_t +# define PyInt_AsLong PyLong_AsLong #endif #if PY_MAJOR_VERSION >= 3 @@ -3404,11 +3407,11 @@ const void **pkey; int err; - key = PyString_FromStringAndSize(NULL, keylength * sizeof(void *)); + key = PyBytes_FromStringAndSize(NULL, keylength * sizeof(void *)); if (key == NULL) goto error; - pkey = (const void **)PyString_AS_STRING(key); + pkey = (const void **)PyBytes_AS_STRING(key); for (i = 0; i < keylength; i++) pkey[i] = unique_key[i]; From noreply at buildbot.pypy.org Wed Apr 29 22:31:34 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 29 Apr 2015 22:31:34 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Fun. Remove this case when testing on 32-bits. Message-ID: <20150429203134.E1F5E1C1334@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1882:0ce3a90f5bb0 Date: 2015-04-29 22:32 +0200 http://bitbucket.org/cffi/cffi/changeset/0ce3a90f5bb0/ Log: Fun. Remove this case when testing on 32-bits. diff --git a/_cffi1/test_recompiler.py b/_cffi1/test_recompiler.py --- a/_cffi1/test_recompiler.py +++ b/_cffi1/test_recompiler.py @@ -178,9 +178,13 @@ def test_macro_check_value(): # the value '-0x80000000' in C sources does not have a clear meaning # to me; it appears to have a different effect than '-2147483648'... + # Moreover, on 32-bits, -2147483648 is actually equal to + # -2147483648U, which in turn is equal to 2147483648U and so positive. vals = ['42', '-42', '0x80000000', '-2147483648', '0', '9223372036854775809ULL', '-9223372036854775807LL'] + if sys.maxsize <= 2**32: + vals.remove('-2147483648') ffi = FFI() cdef_lines = ['#define FOO_%d_%d %s' % (i, j, vals[i]) for i in range(len(vals)) From noreply at buildbot.pypy.org Wed Apr 29 22:37:32 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 29 Apr 2015 22:37:32 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Python 3 compat Message-ID: <20150429203732.AA7C81C1334@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1883:dd6bfefe3e78 Date: 2015-04-29 22:35 +0200 http://bitbucket.org/cffi/cffi/changeset/dd6bfefe3e78/ Log: Python 3 compat diff --git a/_cffi1/test_verify1.py b/_cffi1/test_verify1.py --- a/_cffi1/test_verify1.py +++ b/_cffi1/test_verify1.py @@ -1733,7 +1733,7 @@ (-1, ''), (0xffffffff, 'U'), (maxulong, 'UL'), - (-maxulong / 3, 'L')]: + (-int(maxulong / 3), 'L')]: if c2c and sys.platform == 'win32': continue # enums may always be signed with MSVC ffi = FFI() From noreply at buildbot.pypy.org Wed Apr 29 22:37:33 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 29 Apr 2015 22:37:33 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Python 3 compatible code in these demos Message-ID: <20150429203733.C99071C1334@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1884:e6cfa95230d2 Date: 2015-04-29 22:38 +0200 http://bitbucket.org/cffi/cffi/changeset/e6cfa95230d2/ Log: Python 3 compatible code in these demos diff --git a/demo/bsdopendirtype.py b/demo/bsdopendirtype.py --- a/demo/bsdopendirtype.py +++ b/demo/bsdopendirtype.py @@ -16,10 +16,10 @@ def opendir(dir): if len(dir) == 0: - dir = '.' + dir = b'.' dirname = dir - if not dirname.endswith('/'): - dirname += '/' + if not dirname.endswith(b'/'): + dirname += b'/' dirp = lib.opendir(dir) if dirp == ffi.NULL: raise _posix_error() @@ -32,7 +32,7 @@ raise _posix_error() return name = ffi.string(dirent.d_name) - if name == '.' or name == '..': + if name == b'.' or name == b'..': continue name = dirname + name try: @@ -44,5 +44,5 @@ lib.closedir(dirp) if __name__ == '__main__': - for name, smode in opendir('/tmp'): - print hex(smode), name + for name, smode in opendir(b'/tmp'): + print(hex(smode), name) diff --git a/demo/recopendirtype.py b/demo/recopendirtype.py --- a/demo/recopendirtype.py +++ b/demo/recopendirtype.py @@ -16,10 +16,10 @@ def opendir(dir): if len(dir) == 0: - dir = '.' + dir = b'.' dirname = dir - if not dirname.endswith('/'): - dirname += '/' + if not dirname.endswith(b'/'): + dirname += b'/' dirp = lib.opendir(dir) if dirp == ffi.NULL: raise _posix_error() @@ -34,7 +34,7 @@ if result[0] == ffi.NULL: return # name = ffi.string(dirent.d_name) - if name == '.' or name == '..': + if name == b'.' or name == b'..': continue name = dirname + name try: @@ -46,5 +46,5 @@ lib.closedir(dirp) if __name__ == '__main__': - for name, smode in opendir('/tmp'): - print hex(smode), name + for name, smode in opendir(b'/tmp'): + print(hex(smode), name) From noreply at buildbot.pypy.org Thu Apr 30 00:46:36 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 30 Apr 2015 00:46:36 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Windows-friendlification Message-ID: <20150429224636.E3CBB1C0207@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1885:68e7f03a93d1 Date: 2015-04-30 00:47 +0200 http://bitbucket.org/cffi/cffi/changeset/68e7f03a93d1/ Log: Windows-friendlification diff --git a/_cffi1/cffi1_module.c b/_cffi1/cffi1_module.c --- a/_cffi1/cffi1_module.c +++ b/_cffi1/cffi1_module.c @@ -102,22 +102,22 @@ const struct _cffi_type_context_s *ctx) { PyObject *m; + FFIObject *ffi; + LibObject *lib; #if PY_MAJOR_VERSION >= 3 /* note: the module_def leaks, but anyway the C extension module cannot be unloaded */ - struct PyModuleDef *module_def; - module_def = PyObject_Malloc(sizeof(struct PyModuleDef)); - if (module_def == NULL) - return PyErr_NoMemory(); - - struct PyModuleDef local_module_def = { + struct PyModuleDef *module_def, local_module_def = { PyModuleDef_HEAD_INIT, module_name, NULL, -1, NULL, NULL, NULL, NULL, NULL }; + module_def = PyMem_Malloc(sizeof(struct PyModuleDef)); + if (module_def == NULL) + return PyErr_NoMemory(); *module_def = local_module_def; m = PyModule_Create(module_def); #else @@ -126,12 +126,12 @@ if (m == NULL) return NULL; - FFIObject *ffi = ffi_internal_new(&FFI_Type, ctx); + ffi = ffi_internal_new(&FFI_Type, ctx); Py_XINCREF(ffi); /* make the ffi object really immortal */ if (ffi == NULL || PyModule_AddObject(m, "ffi", (PyObject *)ffi) < 0) return NULL; - LibObject *lib = lib_internal_new(ffi->types_builder, module_name); + lib = lib_internal_new(ffi->types_builder, module_name); if (lib == NULL || PyModule_AddObject(m, "lib", (PyObject *)lib) < 0) return NULL; diff --git a/_cffi1/ffi_obj.c b/_cffi1/ffi_obj.c --- a/_cffi1/ffi_obj.c +++ b/_cffi1/ffi_obj.c @@ -110,6 +110,9 @@ Does not return a new reference! */ if ((accept & ACCEPT_STRING) && PyText_Check(arg)) { + int index, err; + char *input_text; + CTypeDescrObject *ct; PyObject *types_dict = ffi->types_builder->types_dict; PyObject *x = PyDict_GetItem(types_dict, arg); if (x != NULL) { @@ -117,8 +120,8 @@ return (CTypeDescrObject *)x; } - char *input_text = PyText_AS_UTF8(arg); - int index = parse_c_type(&ffi->info, input_text); + input_text = PyText_AS_UTF8(arg); + index = parse_c_type(&ffi->info, input_text); if (index < 0) { size_t num_spaces = ffi->info.error_location; char spaces[num_spaces + 1]; @@ -128,7 +131,6 @@ input_text, spaces); return NULL; } - CTypeDescrObject *ct; if (accept & CONSIDER_FN_AS_FNPTR) { ct = realize_c_type_fn_as_fnptr(ffi->types_builder, ffi->info.output, index); @@ -147,7 +149,7 @@ sure that in any case the next _ffi_type() with the same 'arg' will succeed early, in PyDict_GetItem() above. */ - int err = PyDict_SetItem(types_dict, arg, (PyObject *)ct); + err = PyDict_SetItem(types_dict, arg, (PyObject *)ct); Py_DECREF(ct); /* we know it was written in types_dict (unless we got out of memory), so there is at least this reference left */ if (err < 0) @@ -822,6 +824,8 @@ _fetch_external_struct_or_union(const struct _cffi_struct_union_s *s, PyObject *included_ffis, int recursion) { + Py_ssize_t i; + if (included_ffis == NULL) return NULL; @@ -831,7 +835,6 @@ return NULL; } - Py_ssize_t i; for (i = 0; i < PyTuple_GET_SIZE(included_ffis); i++) { FFIObject *ffi1; const struct _cffi_struct_union_s *s1; diff --git a/_cffi1/lib_obj.c b/_cffi1/lib_obj.c --- a/_cffi1/lib_obj.c +++ b/_cffi1/lib_obj.c @@ -31,6 +31,9 @@ static PyObject *_cpyextfunc_type_index(PyObject *x) { struct CPyExtFunc_s *exf; + LibObject *lib; + PyObject *tuple, *result; + assert(PyErr_Occurred()); if (!PyCFunction_Check(x)) @@ -44,9 +47,7 @@ PyErr_Clear(); - LibObject *lib = (LibObject *)PyCFunction_GET_SELF(x); - PyObject *tuple, *result; - + lib = (LibObject *)PyCFunction_GET_SELF(x); tuple = _realize_c_type_or_func(lib->l_types_builder, lib->l_types_builder->ctx.types, exf->type_index); @@ -84,7 +85,8 @@ by calling _cffi_type(). */ CTypeDescrObject *ct; - int type_index = _CFFI_GETARG(g->type_op); + struct CPyExtFunc_s *xfunc; + int i, type_index = _CFFI_GETARG(g->type_op); _cffi_opcode_t *opcodes = lib->l_types_builder->ctx.types; assert(_CFFI_GETOP(opcodes[type_index]) == _CFFI_OP_FUNCTION); @@ -96,7 +98,7 @@ Py_DECREF(ct); /* argument types: */ - int i = type_index + 1; + i = type_index + 1; while (_CFFI_GETOP(opcodes[i]) != _CFFI_OP_FUNCTION_END) { ct = realize_c_type(lib->l_types_builder, opcodes, i); if (ct == NULL) @@ -110,10 +112,11 @@ There is one per real C function in a CFFI C extension module. CPython never unloads its C extension modules anyway. */ - struct CPyExtFunc_s *xfunc = calloc(1, sizeof(struct CPyExtFunc_s)); + xfunc = PyMem_Malloc(sizeof(struct CPyExtFunc_s)); if (xfunc == NULL) goto no_memory; + memset((char *)xfunc, 0, sizeof(struct CPyExtFunc_s)); xfunc->md.ml_meth = (PyCFunction)g->address; xfunc->md.ml_flags = flags; xfunc->md.ml_name = g->name; @@ -135,15 +138,18 @@ { /* does not return a new reference! */ PyObject *x; - + int index; + const struct _cffi_global_s *g; + CTypeDescrObject *ct; char *s = PyText_AsUTF8(name); if (s == NULL) return NULL; - int index = search_in_globals(&lib->l_types_builder->ctx, s, strlen(s)); + index = search_in_globals(&lib->l_types_builder->ctx, s, strlen(s)); if (index < 0) { if (lib->l_includes != NULL) { + Py_ssize_t i; if (recursion > 100) { PyErr_SetString(PyExc_RuntimeError, @@ -151,7 +157,6 @@ return NULL; } - Py_ssize_t i; for (i = 0; i < PyTuple_GET_SIZE(lib->l_includes); i++) { LibObject *lib1; lib1 = (LibObject *)PyTuple_GET_ITEM(lib->l_includes, i); @@ -180,8 +185,7 @@ return NULL; } - const struct _cffi_global_s *g = &lib->l_types_builder->ctx.globals[index]; - CTypeDescrObject *ct; + g = &lib->l_types_builder->ctx.globals[index]; switch (_CFFI_GETOP(g->type_op)) { @@ -303,13 +307,11 @@ static PyObject *lib_dir(LibObject *lib, PyObject *noarg) { const struct _cffi_global_s *g = lib->l_types_builder->ctx.globals; - int total = lib->l_types_builder->ctx.num_globals; - + int i, total = lib->l_types_builder->ctx.num_globals; PyObject *lst = PyList_New(total); if (lst == NULL) return NULL; - int i; for (i = 0; i < total; i++) { PyObject *s = PyText_FromString(g[i].name); if (s == NULL) { diff --git a/_cffi1/parse_c_type.c b/_cffi1/parse_c_type.c --- a/_cffi1/parse_c_type.c +++ b/_cffi1/parse_c_type.c @@ -225,6 +225,8 @@ type). The 'outer' argument is the index of the opcode outside this "sequel". */ + int check_for_grouping; + _cffi_opcode_t result, *p_current; header: switch (tok->kind) { @@ -244,14 +246,14 @@ break; } - int check_for_grouping = 1; + check_for_grouping = 1; if (tok->kind == TOK_IDENTIFIER) { next_token(tok); /* skip a potential variable name */ check_for_grouping = 0; } - _cffi_opcode_t result = 0; - _cffi_opcode_t *p_current = &result; + result = 0; + p_current = &result; while (tok->kind == TOK_OPEN_PAREN) { next_token(tok); @@ -298,13 +300,15 @@ if (tok->kind != TOK_CLOSE_PAREN) { while (1) { + int arg; + _cffi_opcode_t oarg; + if (tok->kind == TOK_DOTDOTDOT) { has_ellipsis = 1; next_token(tok); break; } - int arg = parse_complete(tok); - _cffi_opcode_t oarg; + arg = parse_complete(tok); switch (_CFFI_GETOP(tok->output[arg])) { case _CFFI_OP_ARRAY: case _CFFI_OP_OPEN_ARRAY: @@ -543,6 +547,10 @@ static int parse_complete(token_t *tok) { + unsigned int t0; + _cffi_opcode_t t1; + int modifiers_length, modifiers_sign; + qualifiers: switch (tok->kind) { case TOK_CONST: @@ -557,10 +565,8 @@ ; } - unsigned int t0; - _cffi_opcode_t t1; - int modifiers_length = 0; - int modifiers_sign = 0; + modifiers_length = 0; + modifiers_sign = 0; modifiers: switch (tok->kind) { @@ -682,12 +688,12 @@ case TOK_STRUCT: case TOK_UNION: { - int kind = tok->kind; + int n, kind = tok->kind; next_token(tok); if (tok->kind != TOK_IDENTIFIER) return parse_error(tok, "struct or union name expected"); - int n = search_in_struct_unions(tok->info->ctx, tok->p, tok->size); + n = search_in_struct_unions(tok->info->ctx, tok->p, tok->size); if (n < 0) return parse_error(tok, "undefined struct/union name"); if (((tok->info->ctx->struct_unions[n].flags & _CFFI_F_UNION) != 0) @@ -699,11 +705,12 @@ } case TOK_ENUM: { + int n; next_token(tok); if (tok->kind != TOK_IDENTIFIER) return parse_error(tok, "enum name expected"); - int n = search_in_enums(tok->info->ctx, tok->p, tok->size); + n = search_in_enums(tok->info->ctx, tok->p, tok->size); if (n < 0) return parse_error(tok, "undefined enum name"); diff --git a/_cffi1/realize_c_type.c b/_cffi1/realize_c_type.c --- a/_cffi1/realize_c_type.c +++ b/_cffi1/realize_c_type.c @@ -89,11 +89,12 @@ static builder_c_t *new_builder_c(const struct _cffi_type_context_s *ctx) { + builder_c_t *builder; PyObject *ldict = PyDict_New(); if (ldict == NULL) return NULL; - builder_c_t *builder = PyMem_Malloc(sizeof(builder_c_t)); + builder = PyMem_Malloc(sizeof(builder_c_t)); if (builder == NULL) { Py_DECREF(ldict); PyErr_NoMemory(); @@ -186,6 +187,7 @@ static PyObject *realize_global_int(const struct _cffi_global_s *g) { + char got[64]; unsigned long long value; /* note: we cast g->address to this function type; we do the same in parse_c_type:parse_sequel() too */ @@ -208,8 +210,6 @@ default: break; } - - char got[64]; if (neg == 2) sprintf(got, "%llu (0x%llx)", value, value); else @@ -236,11 +236,12 @@ return (CTypeDescrObject *)x; } else { + char *text1, *text2; PyObject *y; assert(PyTuple_Check(x)); y = PyTuple_GET_ITEM(x, 0); - char *text1 = ((CTypeDescrObject *)y)->ct_name; - char *text2 = text1 + ((CTypeDescrObject *)y)->ct_name_position + 1; + text1 = ((CTypeDescrObject *)y)->ct_name; + text2 = text1 + ((CTypeDescrObject *)y)->ct_name_position + 1; assert(text2[-3] == '('); text2[-3] = '\0'; PyErr_Format(FFIError, "the type '%s%s' is a function type, not a " @@ -444,15 +445,15 @@ Py_INCREF(x); } else { - PyObject *basetd = get_primitive_type(e->type_prim); - if (basetd == NULL) - return NULL; - PyObject *enumerators = NULL, *enumvalues = NULL, *tmp; Py_ssize_t i, j, n = 0; const char *p; const struct _cffi_global_s *g; int gindex; + PyObject *args; + PyObject *basetd = get_primitive_type(e->type_prim); + if (basetd == NULL) + return NULL; if (*e->enumerators != '\0') { n++; @@ -492,7 +493,7 @@ p += j + 1; } - PyObject *args = NULL; + args = NULL; if (!PyErr_Occurred()) { char *name = alloca(6 + strlen(e->name)); _realize_name(name, "enum ", e->name); @@ -603,30 +604,35 @@ assert(ct->ct_flags & (CT_STRUCT | CT_UNION)); if (ct->ct_flags & CT_LAZY_FIELD_LIST) { + builder_c_t *builder; + char *p; + int n, i, sflags; + const struct _cffi_struct_union_s *s; + const struct _cffi_field_s *fld; + PyObject *fields, *args, *res; + assert(!(ct->ct_flags & CT_IS_OPAQUE)); - builder_c_t *builder = ct->ct_extra; + builder = ct->ct_extra; assert(builder != NULL); - char *p = alloca(2 + strlen(ct->ct_name)); + p = alloca(2 + strlen(ct->ct_name)); _unrealize_name(p, ct->ct_name); - int n = search_in_struct_unions(&builder->ctx, p, strlen(p)); + n = search_in_struct_unions(&builder->ctx, p, strlen(p)); if (n < 0) Py_FatalError("lost a struct/union!"); - const struct _cffi_struct_union_s *s = &builder->ctx.struct_unions[n]; - const struct _cffi_field_s *fld = - &builder->ctx.fields[s->first_field_index]; + s = &builder->ctx.struct_unions[n]; + fld = &builder->ctx.fields[s->first_field_index]; /* XXX painfully build all the Python objects that are the args to b_complete_struct_or_union() */ - PyObject *fields = PyList_New(s->num_fields); + fields = PyList_New(s->num_fields); if (fields == NULL) return -1; - int i; for (i = 0; i < s->num_fields; i++, fld++) { _cffi_opcode_t op = fld->field_type_op; int fbitsize = -1; @@ -674,24 +680,23 @@ PyList_SET_ITEM(fields, i, f); } - int sflags = 0; + sflags = 0; if (s->flags & _CFFI_F_CHECK_FIELDS) sflags |= SF_STD_FIELD_POS; if (s->flags & _CFFI_F_PACKED) sflags |= SF_PACKED; - PyObject *args = Py_BuildValue("(OOOnni)", ct, fields, - Py_None, - (Py_ssize_t)s->size, - (Py_ssize_t)s->alignment, - sflags); + args = Py_BuildValue("(OOOnni)", ct, fields, Py_None, + (Py_ssize_t)s->size, + (Py_ssize_t)s->alignment, + sflags); Py_DECREF(fields); if (args == NULL) return -1; ct->ct_extra = NULL; ct->ct_flags |= CT_IS_OPAQUE; - PyObject *res = b_complete_struct_or_union(NULL, args); + res = b_complete_struct_or_union(NULL, args); ct->ct_flags &= ~CT_IS_OPAQUE; Py_DECREF(args); From noreply at buildbot.pypy.org Thu Apr 30 01:02:14 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 30 Apr 2015 01:02:14 +0200 (CEST) Subject: [pypy-commit] cffi default: More Windows-specific hack Message-ID: <20150429230214.85EAD1C0207@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1886:e77aca5b0519 Date: 2015-04-30 01:00 +0200 http://bitbucket.org/cffi/cffi/changeset/e77aca5b0519/ Log: More Windows-specific hack diff --git a/cffi/ffiplatform.py b/cffi/ffiplatform.py --- a/cffi/ffiplatform.py +++ b/cffi/ffiplatform.py @@ -24,6 +24,7 @@ pass def get_extension(srcfilename, modname, sources=(), **kwds): + _hack_at_distutils() # *before* the following import from distutils.core import Extension allsources = [srcfilename] allsources.extend(sources) From noreply at buildbot.pypy.org Thu Apr 30 01:02:15 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 30 Apr 2015 01:02:15 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: more Windows fixes Message-ID: <20150429230215.859C71C0207@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1887:596f69d5291c Date: 2015-04-30 01:02 +0200 http://bitbucket.org/cffi/cffi/changeset/596f69d5291c/ Log: more Windows fixes diff --git a/_cffi1/ffi_obj.c b/_cffi1/ffi_obj.c --- a/_cffi1/ffi_obj.c +++ b/_cffi1/ffi_obj.c @@ -124,7 +124,7 @@ index = parse_c_type(&ffi->info, input_text); if (index < 0) { size_t num_spaces = ffi->info.error_location; - char spaces[num_spaces + 1]; + char *spaces = alloca(num_spaces + 1); memset(spaces, ' ', num_spaces); spaces[num_spaces] = '\0'; PyErr_Format(FFIError, "%s\n%s\n%s^", ffi->info.error_message, @@ -447,16 +447,16 @@ static PyObject *ffi_getctype(FFIObject *self, PyObject *args) { - PyObject *cdecl, *res; + PyObject *c_decl, *res; char *p, *replace_with = ""; int add_paren, add_space; CTypeDescrObject *ct; size_t replace_with_len; - if (!PyArg_ParseTuple(args, "O|s:getctype", &cdecl, &replace_with)) + if (!PyArg_ParseTuple(args, "O|s:getctype", &c_decl, &replace_with)) return NULL; - ct = _ffi_type(self, cdecl, ACCEPT_STRING|ACCEPT_CTYPE); + ct = _ffi_type(self, c_decl, ACCEPT_STRING|ACCEPT_CTYPE); if (ct == NULL) return NULL; @@ -603,20 +603,20 @@ static PyObject *ffi_callback(FFIObject *self, PyObject *args, PyObject *kwds) { - PyObject *cdecl, *python_callable = Py_None, *error = Py_None; + PyObject *c_decl, *python_callable = Py_None, *error = Py_None; PyObject *res; static char *keywords[] = {"cdecl", "python_callable", "error", NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|OO", keywords, - &cdecl, &python_callable, &error)) + &c_decl, &python_callable, &error)) return NULL; - cdecl = (PyObject *)_ffi_type(self, cdecl, ACCEPT_STRING | ACCEPT_CTYPE | - CONSIDER_FN_AS_FNPTR); - if (cdecl == NULL) + c_decl = (PyObject *)_ffi_type(self, c_decl, ACCEPT_STRING | ACCEPT_CTYPE | + CONSIDER_FN_AS_FNPTR); + if (c_decl == NULL) return NULL; - args = Py_BuildValue("(OOO)", cdecl, python_callable, error); + args = Py_BuildValue("(OOO)", c_decl, python_callable, error); if (args == NULL) return NULL; diff --git a/_cffi1/parse_c_type.c b/_cffi1/parse_c_type.c --- a/_cffi1/parse_c_type.c +++ b/_cffi1/parse_c_type.c @@ -351,9 +351,11 @@ case TOK_INTEGER: errno = 0; +#ifndef MS_WIN32 if (sizeof(length) > sizeof(unsigned long)) length = strtoull(tok->p, &endptr, 0); else +#endif length = strtoul(tok->p, &endptr, 0); if (endptr != tok->p + tok->size) return parse_error(tok, "invalid number"); diff --git a/_cffi1/parse_c_type.h b/_cffi1/parse_c_type.h --- a/_cffi1/parse_c_type.h +++ b/_cffi1/parse_c_type.h @@ -1,5 +1,3 @@ -#include - typedef void *_cffi_opcode_t; @@ -137,7 +135,7 @@ struct _cffi_parse_info_s { const struct _cffi_type_context_s *ctx; _cffi_opcode_t *output; - int output_size; + unsigned int output_size; size_t error_location; const char *error_message; }; diff --git a/cffi/ffiplatform.py b/cffi/ffiplatform.py --- a/cffi/ffiplatform.py +++ b/cffi/ffiplatform.py @@ -24,6 +24,7 @@ pass def get_extension(srcfilename, modname, sources=(), **kwds): + _hack_at_distutils() # *before* the following import from distutils.core import Extension allsources = [srcfilename] allsources.extend(sources) From noreply at buildbot.pypy.org Thu Apr 30 01:20:11 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 30 Apr 2015 01:20:11 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: more Windows fixes Message-ID: <20150429232011.C642E1C03F7@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1888:f6ac55047b65 Date: 2015-04-30 01:20 +0200 http://bitbucket.org/cffi/cffi/changeset/f6ac55047b65/ Log: more Windows fixes diff --git a/_cffi1/_cffi_include.h b/_cffi1/_cffi_include.h --- a/_cffi1/_cffi_include.h +++ b/_cffi1/_cffi_include.h @@ -175,6 +175,12 @@ ((got_nonpos) == (expected <= 0) && \ (got) == (unsigned long long)expected) +#ifdef __GNUC__ +# define _CFFI_UNUSED_FN __attribute__((unused)) +#else +# define _CFFI_UNUSED_FN /* nothing */ +#endif + static int _cffi_init(void) { diff --git a/_cffi1/recompiler.py b/_cffi1/recompiler.py --- a/_cffi1/recompiler.py +++ b/_cffi1/recompiler.py @@ -467,7 +467,7 @@ return prnt = self._prnt checkfuncname = '_cffi_checkfld_%s' % (approxname,) - prnt('__attribute__((unused))') + prnt('_CFFI_UNUSED_FN') prnt('static void %s(%s *p)' % (checkfuncname, cname)) prnt('{') prnt(' /* only to generate compile-time warnings or errors */') diff --git a/_cffi1/test_new_ffi_1.py b/_cffi1/test_new_ffi_1.py --- a/_cffi1/test_new_ffi_1.py +++ b/_cffi1/test_new_ffi_1.py @@ -74,7 +74,13 @@ DEFS_PACKED = """ struct is_packed { char a; int b; } /*here*/; """ - CCODE = DEFS + DEFS_PACKED.replace('/*here*/', '__attribute__((packed))') + if sys.platform == "win32": + DEFS = DEFS.replace('data[0]', 'data[1]') # not supported + CCODE = (DEFS + "\n#pragma pack(push,1)\n" + DEFS_PACKED + + "\n#pragma pack(pop)\n") + else: + CCODE = (DEFS + + DEFS_PACKED.replace('/*here*/', '__attribute__((packed))')) ffi1.cdef(DEFS) ffi1.cdef(DEFS_PACKED, packed=True) @@ -884,7 +890,8 @@ assert ffi.cast("enum bar", 0) != ffi.cast("int", 0) assert repr(ffi.cast("enum bar", -1)) == "" assert repr(ffi.cast("enum foq", -1)) == ( # enums are unsigned, if - "") # they contain no neg value + "") or ( # they contain no neg value + sys.platform == "win32") # (but not on msvc) # enum baz { A2=0x1000, B2=0x2000 }; assert ffi.string(ffi.cast("enum baz", 0x1000)) == "A2" assert ffi.string(ffi.cast("enum baz", 0x2000)) == "B2" @@ -902,8 +909,8 @@ assert s.e == 2 assert s[0].e == 2 s.e = ffi.cast("enum foo2", -1) - assert s.e == 4294967295 - assert s[0].e == 4294967295 + assert s.e in (4294967295, -1) # two choices + assert s[0].e in (4294967295, -1) s.e = s.e py.test.raises(TypeError, "s.e = 'B3'") py.test.raises(TypeError, "s.e = '2'") @@ -1000,6 +1007,8 @@ def test_bitfield_enum(self): # typedef enum { AA1, BB1, CC1 } foo_e_t; # typedef struct { foo_e_t f:2; } bfenum_t; + if sys.platform == "win32": + py.test.skip("enums are not unsigned") s = ffi.new("bfenum_t *") s.f = 2 assert s.f == 2 @@ -1198,6 +1207,8 @@ assert repr(p.a).startswith(" Author: Armin Rigo Branch: cffi-1.0 Changeset: r1889:1d37117fd28c Date: 2015-04-30 01:22 +0200 http://bitbucket.org/cffi/cffi/changeset/1d37117fd28c/ Log: hg merge default diff --git a/doc/source/index.rst b/doc/source/index.rst --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -437,9 +437,9 @@ types ``TBYTE TCHAR LPCTSTR PCTSTR LPTSTR PTSTR PTBYTE PTCHAR`` are no longer automatically defined; see ``ffi.set_unicode()`` below. -* *New in version 0.9:* the other standard integer types from stdint.h, +* *New in version 0.9.3:* the other standard integer types from stdint.h, as long as they map to integers of 1, 2, 4 or 8 bytes. Larger integers - are not supported. + are not supported. (Actually added in version 0.9 but this was buggy.) .. _`common Windows types`: http://msdn.microsoft.com/en-us/library/windows/desktop/aa383751%28v=vs.85%29.aspx @@ -1078,6 +1078,23 @@ C.printf("hello, %f\n", ffi.cast("double", 42)) C.printf("hello, %s\n", ffi.new("char[]", "world")) +Note that if you are using ``dlopen()``, the function declaration in the +``cdef()`` must match the original one in C exactly, as usual --- in +particular, if this function is variadic in C, then its ``cdef()`` +declaration must also be variadic. You cannot declare it in the +``cdef()`` with fixed arguments instead, even if you plan to only call +it with these argument types. The reason is that some architectures +have a different calling convention depending on whether the function +signature is fixed or not. (On x86-64, the difference can sometimes be +seen in PyPy's JIT-generated code if some arguments are ``double``.) + +Note that the function signature ``int foo();`` is interpreted by CFFI +as equivalent to ``int foo(void);``. This differs from the C standard, +in which ``int foo();`` is really like ``int foo(...);`` and can be +called with any arguments. (This feature of C is a pre-C89 relic: the +arguments cannot be accessed at all in the body of ``foo()`` without +relying on compiler-specific extensions.) + Callbacks --------- diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ p = subprocess.Popen([pkg_config, option, 'libffi'], stdout=subprocess.PIPE) except OSError as e: - if e.errno != errno.ENOENT: + if e.errno not in [errno.ENOENT, errno.EACCES]: raise else: t = p.stdout.read().decode().strip() From noreply at buildbot.pypy.org Thu Apr 30 09:30:25 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 30 Apr 2015 09:30:25 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: setuptools_ext: generate the C module in memory. Avoids the os.rename() Message-ID: <20150430073025.B449C1C0207@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1890:bae04f21862d Date: 2015-04-30 09:19 +0200 http://bitbucket.org/cffi/cffi/changeset/bae04f21862d/ Log: setuptools_ext: generate the C module in memory. Avoids the os.rename() which, I think, fails to overwrite on Windows. diff --git a/_cffi1/recompiler.py b/_cffi1/recompiler.py --- a/_cffi1/recompiler.py +++ b/_cffi1/recompiler.py @@ -1,4 +1,4 @@ -import os, sys +import os, sys, io from cffi import ffiplatform, model from .cffi_opcode import * @@ -806,11 +806,26 @@ self.cffi_types[index] = CffiOp(OP_ENUM, enum_index) -def make_c_source(ffi, module_name, preamble, target_c_file): +if sys.version_info >= (3,): + NativeIO = io.StringIO +else: + class NativeIO(io.BytesIO): + def write(self, s): + if isinstance(s, unicode): + s = s.encode('ascii') + super(NativeIO, self).write(s) + +def make_c_source(ffi, module_name, preamble, target_c_file=NativeIO): recompiler = Recompiler(ffi, module_name) recompiler.collect_type_table() - with open(target_c_file, 'w') as f: + if target_c_file is NativeIO: + f = NativeIO() recompiler.write_source_to_f(f, preamble) + return f.getvalue() + else: + with open(target_c_file, 'w') as f: + recompiler.write_source_to_f(f, preamble) + return None def _get_extension(module_name, c_file, kwds): source_name = ffiplatform.maybe_relative_path(c_file) diff --git a/_cffi1/setuptools_ext.py b/_cffi1/setuptools_ext.py --- a/_cffi1/setuptools_ext.py +++ b/_cffi1/setuptools_ext.py @@ -40,22 +40,20 @@ ext = Extension(name=module_name, sources=allsources, **kwds) def make_mod(tmpdir): - mkpath(tmpdir) file_name = module_name + '.c' log.info("generating cffi module %r" % file_name) + output = recompiler.make_c_source(ffi, module_name, source) + mkpath(tmpdir) c_file = os.path.join(tmpdir, file_name) - c_tmp = '%s.%s' % (c_file, os.getpid()) - recompiler.make_c_source(ffi, module_name, source, c_tmp) try: with open(c_file, 'r') as f1: - with open(c_tmp, 'r') as f2: - if f1.read() != f2.read(): - raise IOError + if f1.read() != output: + raise IOError except IOError: - os.rename(c_tmp, c_file) + with open(c_file, 'w') as f1: + f1.write(output) else: log.info("already up-to-date") - os.unlink(c_tmp) return c_file if dist.ext_modules is None: From noreply at buildbot.pypy.org Thu Apr 30 09:30:26 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 30 Apr 2015 09:30:26 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Try to support unicode_literals Message-ID: <20150430073026.B96351C0207@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1891:4480292812da Date: 2015-04-30 09:30 +0200 http://bitbucket.org/cffi/cffi/changeset/4480292812da/ Log: Try to support unicode_literals diff --git a/_cffi1/recompiler.py b/_cffi1/recompiler.py --- a/_cffi1/recompiler.py +++ b/_cffi1/recompiler.py @@ -832,6 +832,8 @@ return ffiplatform.get_extension(source_name, module_name, **kwds) def recompile(ffi, module_name, preamble, tmpdir='.', **kwds): + if not isinstance(module_name, str): + module_name = module_name.encode('ascii') c_file = os.path.join(tmpdir, module_name + '.c') ext = _get_extension(module_name, c_file, kwds) make_c_source(ffi, module_name, preamble, c_file) diff --git a/_cffi1/test_unicode_literals.py b/_cffi1/test_unicode_literals.py new file mode 100644 --- /dev/null +++ b/_cffi1/test_unicode_literals.py @@ -0,0 +1,86 @@ +# +# ---------------------------------------------- +# WARNING, ALL LITERALS IN THIS FILE ARE UNICODE +# ---------------------------------------------- +# +from __future__ import unicode_literals +# +# +# +import sys, math +from cffi import FFI +from _cffi1 import recompiler + +lib_m = "m" +if sys.platform == 'win32': + #there is a small chance this fails on Mingw via environ $CC + import distutils.ccompiler + if distutils.ccompiler.get_default_compiler() == 'msvc': + lib_m = 'msvcrt' + + +def test_cast(): + ffi = FFI() + assert int(ffi.cast("int", 3.14)) == 3 # unicode literal + +def test_new(): + ffi = FFI() + assert ffi.new("int[]", [3, 4, 5])[2] == 5 # unicode literal + +def test_typeof(): + ffi = FFI() + tp = ffi.typeof("int[51]") # unicode literal + assert tp.length == 51 + +def test_sizeof(): + ffi = FFI() + assert ffi.sizeof("int[51]") == 51 * 4 # unicode literal + +def test_alignof(): + ffi = FFI() + assert ffi.alignof("int[51]") == 4 # unicode literal + +def test_getctype(): + ffi = FFI() + assert ffi.getctype("int**") == "int * *" # unicode literal + assert type(ffi.getctype("int**")) is str + +def test_cdef(): + ffi = FFI() + ffi.cdef("typedef int foo_t[50];") # unicode literal + +def test_offsetof(): + ffi = FFI() + ffi.cdef("typedef struct { int x, y; } foo_t;") + assert ffi.offsetof("foo_t", "y") == 4 # unicode literal + +def test_enum(): + ffi = FFI() + ffi.cdef("enum foo_e { AA, BB, CC };") # unicode literal + x = ffi.cast("enum foo_e", 1) + assert int(ffi.cast("int", x)) == 1 + +def test_dlopen(): + ffi = FFI() + ffi.cdef("double sin(double x);") + m = ffi.dlopen(lib_m) # unicode literal + x = m.sin(1.23) + assert x == math.sin(1.23) + +def test_verify(): + ffi = FFI() + ffi.cdef("double test_verify_1(double x);") # unicode literal + lib = ffi.verify("double test_verify_1(double x) { return x * 42.0; }") + assert lib.test_verify_1(-1.5) == -63.0 + +def test_callback(): + ffi = FFI() + cb = ffi.callback("int(int)", # unicode literal + lambda x: x + 42) + assert cb(5) == 47 + +def test_math_sin_unicode(): + ffi = FFI() + ffi.cdef("float sin(double); double cos(double);") + lib = recompiler.verify(ffi, 'test_math_sin_unicode', '#include ') + assert lib.cos(1.43) == math.cos(1.43) From noreply at buildbot.pypy.org Thu Apr 30 09:30:59 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 30 Apr 2015 09:30:59 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: this test works now Message-ID: <20150430073059.AF92D1C0207@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1892:7395fc0d48c2 Date: 2015-04-30 09:31 +0200 http://bitbucket.org/cffi/cffi/changeset/7395fc0d48c2/ Log: this test works now diff --git a/_cffi1/test_verify1.py b/_cffi1/test_verify1.py --- a/_cffi1/test_verify1.py +++ b/_cffi1/test_verify1.py @@ -1670,7 +1670,6 @@ "function myfunc: 'foo_t' is used as result type, but is opaque") def test_include(): - py.test.xfail("test_include") ffi1 = FFI() ffi1.cdef("typedef struct { int x; ...; } foo_t;") ffi1.verify("typedef struct { int y, x; } foo_t;") From noreply at buildbot.pypy.org Thu Apr 30 09:38:52 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 30 Apr 2015 09:38:52 +0200 (CEST) Subject: [pypy-commit] cffi default: Acquire the import lock explicitly around the sys.setdlopenflags() Message-ID: <20150430073852.751DA1C0207@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1893:5df67a682cac Date: 2015-04-30 09:39 +0200 http://bitbucket.org/cffi/cffi/changeset/5df67a682cac/ Log: Acquire the import lock explicitly around the sys.setdlopenflags() diff --git a/cffi/vengine_cpy.py b/cffi/vengine_cpy.py --- a/cffi/vengine_cpy.py +++ b/cffi/vengine_cpy.py @@ -141,19 +141,23 @@ def load_library(self, flags=None): # XXX review all usages of 'self' here! # import it as a new extension module - if hasattr(sys, "getdlopenflags"): - previous_flags = sys.getdlopenflags() + imp.acquire_lock() try: - if hasattr(sys, "setdlopenflags") and flags is not None: - sys.setdlopenflags(flags) - module = imp.load_dynamic(self.verifier.get_module_name(), - self.verifier.modulefilename) - except ImportError as e: - error = "importing %r: %s" % (self.verifier.modulefilename, e) - raise ffiplatform.VerificationError(error) + if hasattr(sys, "getdlopenflags"): + previous_flags = sys.getdlopenflags() + try: + if hasattr(sys, "setdlopenflags") and flags is not None: + sys.setdlopenflags(flags) + module = imp.load_dynamic(self.verifier.get_module_name(), + self.verifier.modulefilename) + except ImportError as e: + error = "importing %r: %s" % (self.verifier.modulefilename, e) + raise ffiplatform.VerificationError(error) + finally: + if hasattr(sys, "setdlopenflags"): + sys.setdlopenflags(previous_flags) finally: - if hasattr(sys, "setdlopenflags"): - sys.setdlopenflags(previous_flags) + imp.release_lock() # # call loading_cpy_struct() to get the struct layout inferred by # the C compiler From noreply at buildbot.pypy.org Thu Apr 30 09:44:06 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 30 Apr 2015 09:44:06 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: hg merge default Message-ID: <20150430074406.F08301C0207@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1894:d4d01f78d16e Date: 2015-04-30 09:42 +0200 http://bitbucket.org/cffi/cffi/changeset/d4d01f78d16e/ Log: hg merge default diff --git a/cffi/vengine_cpy.py b/cffi/vengine_cpy.py --- a/cffi/vengine_cpy.py +++ b/cffi/vengine_cpy.py @@ -141,19 +141,23 @@ def load_library(self, flags=None): # XXX review all usages of 'self' here! # import it as a new extension module - if hasattr(sys, "getdlopenflags"): - previous_flags = sys.getdlopenflags() + imp.acquire_lock() try: - if hasattr(sys, "setdlopenflags") and flags is not None: - sys.setdlopenflags(flags) - module = imp.load_dynamic(self.verifier.get_module_name(), - self.verifier.modulefilename) - except ImportError as e: - error = "importing %r: %s" % (self.verifier.modulefilename, e) - raise ffiplatform.VerificationError(error) + if hasattr(sys, "getdlopenflags"): + previous_flags = sys.getdlopenflags() + try: + if hasattr(sys, "setdlopenflags") and flags is not None: + sys.setdlopenflags(flags) + module = imp.load_dynamic(self.verifier.get_module_name(), + self.verifier.modulefilename) + except ImportError as e: + error = "importing %r: %s" % (self.verifier.modulefilename, e) + raise ffiplatform.VerificationError(error) + finally: + if hasattr(sys, "setdlopenflags"): + sys.setdlopenflags(previous_flags) finally: - if hasattr(sys, "setdlopenflags"): - sys.setdlopenflags(previous_flags) + imp.release_lock() # # call loading_cpy_struct() to get the struct layout inferred by # the C compiler From noreply at buildbot.pypy.org Thu Apr 30 09:44:08 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 30 Apr 2015 09:44:08 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: Forces setdlopenflags() explicitly Message-ID: <20150430074408.1246F1C0207@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1895:705f395febf0 Date: 2015-04-30 09:44 +0200 http://bitbucket.org/cffi/cffi/changeset/705f395febf0/ Log: Forces setdlopenflags() explicitly diff --git a/_cffi1/test_verify1.py b/_cffi1/test_verify1.py --- a/_cffi1/test_verify1.py +++ b/_cffi1/test_verify1.py @@ -2081,22 +2081,27 @@ assert ffi.getwinerror()[0] == n def test_verify_dlopen_flags(): - py.test.xfail("dlopen flags") + if not hasattr(sys, 'setdlopenflags'): + py.test.skip("requires sys.setdlopenflags()") # Careful with RTLD_GLOBAL. If by chance the FFI is not deleted # promptly, like on PyPy, then other tests may see the same # exported symbols as well. So we must not export a simple name # like 'foo'! - ffi1 = FFI() - ffi1.cdef("int foo_verify_dlopen_flags;") + old = sys.getdlopenflags() + try: + ffi1 = FFI() + ffi1.cdef("int foo_verify_dlopen_flags;") - lib1 = ffi1.verify("int foo_verify_dlopen_flags;", - flags=ffi1.RTLD_GLOBAL | ffi1.RTLD_LAZY) - lib2 = get_second_lib() + sys.setdlopenflags(ffi1.RTLD_GLOBAL | ffi1.RTLD_LAZY) + lib1 = ffi1.verify("int foo_verify_dlopen_flags;") + lib2 = get_second_lib() - lib1.foo_verify_dlopen_flags = 42 - assert lib2.foo_verify_dlopen_flags == 42 - lib2.foo_verify_dlopen_flags += 1 - assert lib1.foo_verify_dlopen_flags == 43 + lib1.foo_verify_dlopen_flags = 42 + assert lib2.foo_verify_dlopen_flags == 42 + lib2.foo_verify_dlopen_flags += 1 + assert lib1.foo_verify_dlopen_flags == 43 + finally: + sys.setdlopenflags(old) def get_second_lib(): # Hack, using modulename makes the test fail From noreply at buildbot.pypy.org Thu Apr 30 11:19:06 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 30 Apr 2015 11:19:06 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: tweaks tweaks Message-ID: <20150430091906.1A00B1C1481@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1897:f5beaf1559c2 Date: 2015-04-30 11:19 +0200 http://bitbucket.org/cffi/cffi/changeset/f5beaf1559c2/ Log: tweaks tweaks diff --git a/MANIFEST.in b/MANIFEST.in --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,5 +3,5 @@ recursive-include testing *.py recursive-include doc *.py *.rst Makefile *.bat recursive-include demo py.cleanup *.py -recursive-include _cffi1 *.py *.h +recursive-include _cffi1 *.py *.c *.h include LICENSE setup_base.py diff --git a/demo/_curses_setup.py b/demo/_curses_setup.py --- a/demo/_curses_setup.py +++ b/demo/_curses_setup.py @@ -4,10 +4,10 @@ name="_curses", version="0.1", py_modules=["_curses"], - setup_requires=["cffi>=1.0"], + setup_requires=["cffi>=1.0.dev0"], cffi_modules=[ "_curses_build:ffi", ], - install_requires=["cffi>=1.0"], # should maybe be "cffi-backend" only? + install_requires=["cffi>=1.0.dev0"], # should maybe be "cffi-backend" only? zip_safe=False, ) diff --git a/demo/bsdopendirtype_setup.py b/demo/bsdopendirtype_setup.py --- a/demo/bsdopendirtype_setup.py +++ b/demo/bsdopendirtype_setup.py @@ -4,10 +4,10 @@ name="example", version="0.1", py_modules=["bsdopendirtype"], - setup_requires=["cffi>=1.0"], + setup_requires=["cffi>=1.0.dev0"], cffi_modules=[ "bsdopendirtype_build:ffi", ], - install_requires=["cffi>=1.0"], # should maybe be "cffi-backend" only? + install_requires=["cffi>=1.0.dev0"], # should maybe be "cffi-backend" only? zip_safe=False, ) diff --git a/demo/readdir2_setup.py b/demo/readdir2_setup.py --- a/demo/readdir2_setup.py +++ b/demo/readdir2_setup.py @@ -4,10 +4,10 @@ name="readdir2", version="0.1", py_modules=["readdir2"], - setup_requires=["cffi>=1.0"], + setup_requires=["cffi>=1.0.dev0"], cffi_modules=[ "readdir2_build:ffi", ], - install_requires=["cffi>=1.0"], # should maybe be "cffi-backend" only? + install_requires=["cffi>=1.0.dev0"], # should maybe be "cffi-backend" only? zip_safe=False, ) diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -142,7 +142,7 @@ `Mailing list `_ """, - version='1.0.dev0', + version='1.0.dev2', packages=['cffi', '_cffi1'], package_data={'_cffi1': ['_cffi_include.h', 'parse_c_type.h']}, zip_safe=False, From noreply at buildbot.pypy.org Thu Apr 30 11:19:04 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 30 Apr 2015 11:19:04 +0200 (CEST) Subject: [pypy-commit] cffi cffi-1.0: updates Message-ID: <20150430091904.E59501C1481@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cffi-1.0 Changeset: r1896:abe9ab7b9b11 Date: 2015-04-30 11:12 +0200 http://bitbucket.org/cffi/cffi/changeset/abe9ab7b9b11/ Log: updates diff --git a/MANIFEST.in b/MANIFEST.in --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,4 +3,5 @@ recursive-include testing *.py recursive-include doc *.py *.rst Makefile *.bat recursive-include demo py.cleanup *.py +recursive-include _cffi1 *.py *.h include LICENSE setup_base.py diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -142,7 +142,7 @@ `Mailing list `_ """, - version='1.0.0', + version='1.0.dev0', packages=['cffi', '_cffi1'], package_data={'_cffi1': ['_cffi_include.h', 'parse_c_type.h']}, zip_safe=False, From noreply at buildbot.pypy.org Thu Apr 30 13:53:10 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Thu, 30 Apr 2015 13:53:10 +0200 (CEST) Subject: [pypy-commit] pypy can_cast: fix translation Message-ID: <20150430115311.013521C0207@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: can_cast Changeset: r76952:af8b59de3288 Date: 2015-04-30 12:52 +0100 http://bitbucket.org/pypy/pypy/changeset/af8b59de3288/ Log: fix translation diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -2494,35 +2494,37 @@ smaller_types = unrolling_iterable( [(tp, tp.Unsigned) for tp in smaller_types]) def min_dtype(self): + value = rffi.cast(UInt64.T, self.value) for Small, USmall in smaller_types: - signed_max = rffi.cast(UInt_t.T, Small.max_value) - unsigned_max = rffi.cast(UInt_t.T, USmall.max_value) - if self.value <= unsigned_max: - if self.value <= signed_max: + signed_max = rffi.cast(UInt64.T, Small.max_value) + unsigned_max = rffi.cast(UInt64.T, USmall.max_value) + if value <= unsigned_max: + if value <= signed_max: return Small.num, USmall.num else: return USmall.num, USmall.num - if self.value <= rffi.cast(UInt_t.T, Int_t.max_value): + if value <= rffi.cast(UInt64.T, Int_t.max_value): return Int_t.num, UInt_t.num else: return UInt_t.num, UInt_t.num UInt_t.BoxType.min_dtype = min_dtype def min_dtype(self): - if self.value >= 0: + value = rffi.cast(Int64.T, self.value) + if value >= 0: for Small, USmall in smaller_types: - signed_max = rffi.cast(UInt_t.T, Small.max_value) - unsigned_max = rffi.cast(UInt_t.T, USmall.max_value) - if self.value <= unsigned_max: - if self.value <= signed_max: + signed_max = rffi.cast(Int64.T, Small.max_value) + unsigned_max = rffi.cast(Int64.T, USmall.max_value) + if value <= unsigned_max: + if value <= signed_max: return Small.num, USmall.num else: return USmall.num, USmall.num return Int_t.num, UInt_t.num else: for Small, USmall in smaller_types: - signed_min = rffi.cast(UInt_t.T, Small.min_value) - if self.value >= signed_max: + signed_min = rffi.cast(Int64.T, Small.min_value) + if value >= signed_min: return Small.num, Small.num return Int_t.num, Int_t.num Int_t.BoxType.min_dtype = min_dtype From noreply at buildbot.pypy.org Thu Apr 30 18:50:03 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Thu, 30 Apr 2015 18:50:03 +0200 (CEST) Subject: [pypy-commit] pypy can_cast: Implement min_dtype() on float types Message-ID: <20150430165003.A350C1C0EC0@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: can_cast Changeset: r76953:244204b77d90 Date: 2015-04-30 17:38 +0100 http://bitbucket.org/pypy/pypy/changeset/244204b77d90/ Log: Implement min_dtype() on float types diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -4053,3 +4053,8 @@ assert np.can_cast(127, np.int8) assert not np.can_cast(128, np.int8) assert np.can_cast(128, np.int16) + + assert np.can_cast(np.float32('inf'), np.float32) + assert np.can_cast(float('inf'), np.float32) # XXX: False in CNumPy?! + assert np.can_cast(3.3e38, np.float32) + assert not np.can_cast(3.4e38, np.float32) diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -1026,6 +1026,7 @@ kind = NPY.FLOATINGLTR char = NPY.HALFLTR BoxType = boxes.W_Float16Box + max_value = 65000. @specialize.argtype(1) def box(self, value): @@ -1070,6 +1071,7 @@ char = NPY.FLOATLTR BoxType = boxes.W_Float32Box format_code = "f" + max_value = 3.4e38 class Float64(BaseType, Float): T = rffi.DOUBLE @@ -1078,6 +1080,7 @@ char = NPY.DOUBLELTR BoxType = boxes.W_Float64Box format_code = "d" + max_value = 1.7e308 class ComplexFloating(object): _mixin_ = True @@ -2487,8 +2490,7 @@ signed_types = [Int8, Int16, Int32, Int64, Long] -for Int_t in signed_types: - UInt_t = Int_t.Unsigned +def make_integer_min_dtype(Int_t, UInt_t): smaller_types = [tp for tp in signed_types if rffi.sizeof(tp.T) < rffi.sizeof(Int_t.T)] smaller_types = unrolling_iterable( @@ -2528,3 +2530,32 @@ return Small.num, Small.num return Int_t.num, Int_t.num Int_t.BoxType.min_dtype = min_dtype + +for Int_t in signed_types: + UInt_t = Int_t.Unsigned + make_integer_min_dtype(Int_t, UInt_t) + + +smaller_float_types = { + Float16: [], Float32: [Float16], Float64: [Float16, Float32], + FloatLong: [Float16, Float32, Float64]} + +def make_float_min_dtype(Float_t): + smaller_types = unrolling_iterable(smaller_float_types[Float_t]) + smallest_type = Float16 + def min_dtype(self): + value = float(self.value) + if not rfloat.isfinite(value): + tp = smallest_type + else: + for SmallFloat in smaller_types: + if -SmallFloat.max_value < value < SmallFloat.max_value: + tp = SmallFloat + break + else: + tp = Float_t + return tp.num, tp.num + Float_t.BoxType.min_dtype = min_dtype + +for Float_t in float_types: + make_float_min_dtype(Float_t) From noreply at buildbot.pypy.org Thu Apr 30 19:13:47 2015 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 30 Apr 2015 19:13:47 +0200 (CEST) Subject: [pypy-commit] pypy default: document fields on pyframe Message-ID: <20150430171347.CCCF41C0207@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r76954:f726f9ee2155 Date: 2015-04-30 19:13 +0200 http://bitbucket.org/pypy/pypy/changeset/f726f9ee2155/ Log: document fields on pyframe diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py --- a/pypy/interpreter/pyframe.py +++ b/pypy/interpreter/pyframe.py @@ -49,14 +49,29 @@ last_instr = -1 last_exception = None f_backref = jit.vref_None + # For tracing w_f_trace = None - # For tracing instr_lb = 0 instr_ub = 0 instr_prev_plus_one = 0 + # end of tracing + is_being_profiled = False escaped = False # see mark_as_escaped() + w_globals = None + w_locals = None # dict containing locals, if forced or necessary + pycode = None # code object executed by that frame + locals_stack_w = None # the list of all locals and valuestack + valuestackdepth = -1 # number of items on valuestack + lastblock = None + builtin = None # builtin cache, only if honor__builtins__ is True, + # default to False + f_lineno = -1 # current lineno + cells = None # cells + + # there is also self.space which is removed by the annotator + def __init__(self, space, code, w_globals, outer_func): if not we_are_translated(): assert type(self) == space.FrameClass, ( @@ -65,11 +80,9 @@ assert isinstance(code, pycode.PyCode) self.space = space self.w_globals = w_globals - self.w_locals = None self.pycode = code self.locals_stack_w = [None] * (code.co_nlocals + code.co_stacksize) self.valuestackdepth = code.co_nlocals - self.lastblock = None make_sure_not_resized(self.locals_stack_w) check_nonneg(self.valuestackdepth) # From noreply at buildbot.pypy.org Thu Apr 30 19:20:13 2015 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 30 Apr 2015 19:20:13 +0200 (CEST) Subject: [pypy-commit] pypy default: document two missing fields Message-ID: <20150430172013.93E1A1C0207@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r76955:b4e74fa7ca14 Date: 2015-04-30 19:20 +0200 http://bitbucket.org/pypy/pypy/changeset/b4e74fa7ca14/ Log: document two missing fields diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py --- a/pypy/interpreter/pyframe.py +++ b/pypy/interpreter/pyframe.py @@ -65,13 +65,19 @@ locals_stack_w = None # the list of all locals and valuestack valuestackdepth = -1 # number of items on valuestack lastblock = None - builtin = None # builtin cache, only if honor__builtins__ is True, # default to False f_lineno = -1 # current lineno cells = None # cells + # other fields: + + # builtin - builtin cache, only if honor__builtins__ is True, + # there is also self.space which is removed by the annotator + # additionally JIT uses vable_token field that is representing + # frame current virtualizable state as seen by the JIT + def __init__(self, space, code, w_globals, outer_func): if not we_are_translated(): assert type(self) == space.FrameClass, ( From noreply at buildbot.pypy.org Thu Apr 30 22:12:06 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 30 Apr 2015 22:12:06 +0200 (CEST) Subject: [pypy-commit] pypy numpy-fixes: preserve stride order in astype Message-ID: <20150430201206.6F37F1C0D78@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: numpy-fixes Changeset: r76957:e7d09eb72b19 Date: 2015-04-30 04:17 +0300 http://bitbucket.org/pypy/pypy/changeset/e7d09eb72b19/ Log: preserve stride order in astype diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py --- a/pypy/module/micronumpy/concrete.py +++ b/pypy/module/micronumpy/concrete.py @@ -329,7 +329,11 @@ return ArrayBuffer(self, readonly) def astype(self, space, dtype): - strides, backstrides = calc_strides(self.get_shape(), dtype, self.order) + s_elsize = self.dtype.elsize + t_elsize = dtype.elsize + strides = [s*t_elsize/s_elsize for s in self.get_strides()] + backstrides = calc_backstrides(strides, self.get_shape()) + #strides, backstrides = calc_strides(self.get_shape(), dtype, self.order) impl = ConcreteArray(self.get_shape(), dtype, self.order, strides, backstrides) loop.setslice(space, impl.get_shape(), impl, self) From noreply at buildbot.pypy.org Thu Apr 30 22:12:05 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 30 Apr 2015 22:12:05 +0200 (CEST) Subject: [pypy-commit] pypy numpy-fixes: test, fix array() Message-ID: <20150430201205.473581C0D78@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: numpy-fixes Changeset: r76956:80fe68674c7a Date: 2015-04-30 04:03 +0300 http://bitbucket.org/pypy/pypy/changeset/80fe68674c7a/ Log: test, fix array() diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py --- a/pypy/module/micronumpy/ctors.py +++ b/pypy/module/micronumpy/ctors.py @@ -124,12 +124,13 @@ copy = True if copy: shape = w_object.get_shape() - elems_w = [None] * w_object.get_size() - elsize = w_object.get_dtype().elsize - # TODO - use w_object.implementation without copying to a list - # unfortunately that causes a union error in translation - for i in range(w_object.get_size()): - elems_w[i] = w_object.implementation.getitem(i * elsize) + w_arr = W_NDimArray.from_shape(space, shape, dtype, order=order) + if support.product(shape) == 1: + w_arr.set_scalar_value(dtype.coerce(space, + w_object.implementation.getitem(0))) + else: + loop.setslice(space, shape, w_arr.implementation, w_object.implementation) + return w_arr else: imp = w_object.implementation with imp as storage: diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -237,7 +237,7 @@ assert np.WRAP is 1 assert np.RAISE is 2 - def test_ndarray(self): + def test_creation(self): from numpy import ndarray, array, dtype, flatiter assert type(ndarray) is type @@ -269,6 +269,12 @@ assert a.flags['C'] assert not a.flags['F'] + x = array([[0, 2], [1, 1], [2, 0]]) + y = array(x.T, dtype=float) + assert (y == x.T).all() + y = array(x.T, copy=False) + assert (y == x.T).all() + def test_ndmin(self): from numpy import array From noreply at buildbot.pypy.org Thu Apr 30 22:12:07 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 30 Apr 2015 22:12:07 +0200 (CEST) Subject: [pypy-commit] pypy numpy-fixes: test, fix creation of record array with ndarray data Message-ID: <20150430201207.90E781C0D78@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: numpy-fixes Changeset: r76958:d7e26a6bffd3 Date: 2015-04-30 22:09 +0300 http://bitbucket.org/pypy/pypy/changeset/d7e26a6bffd3/ Log: test, fix creation of record array with ndarray data diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py --- a/pypy/module/micronumpy/loop.py +++ b/pypy/module/micronumpy/loop.py @@ -209,7 +209,7 @@ while not target_iter.done(target_state): setslice_driver.jit_merge_point(shapelen=shapelen, dtype=dtype) val = source_iter.getitem(source_state) - if dtype.is_str_or_unicode(): + if dtype.is_str_or_unicode() or dtype.is_record(): val = dtype.coerce(space, val) else: val = val.convert_to(space, dtype) diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -3838,6 +3838,14 @@ ([4, 5, 6], [5.5, 6.5, 7.5, 8.5, 9.5])], dtype=d) assert len(list(a[0])) == 2 + + mdtype = dtype([('a', bool), ('b', bool), ('c', bool)]) + a = array([0, 0, 0, 1, 1]) + # this creates a value of (x, x, x) in b for each x in a + b = array(a, dtype=mdtype) + assert b.shape == a.shape + c = array([(x, x, x) for x in [0, 0, 0, 1, 1]], dtype=mdtype) + assert (b == c).all() def test_3d_record(self): from numpy import dtype, array

Coverage report: - 95% -

Coverage for logger : - 96% -

- 46 statements - 44 run - 2 missing - 0 excluded - -

Coverage for rstrategies : - 95% -

- 349 statements - 333 run - 16 missing - 0 excluded - -